diff --git a/PlayWright/scraper.py b/PlayWright/scraper.py new file mode 100644 index 0000000..827ecf5 --- /dev/null +++ b/PlayWright/scraper.py @@ -0,0 +1,487 @@ +""" +scraper.py — Instacart pipeline scraper +========================================== + +Architecture +------------ + 1. Load credentials from CS178-Shopwise/.env + SUPABASE_URL, SUPABASE_SERVICE_KEY + + 2. Fetch all rows from the Supabase `taxonomy` table (column: `ingredient`) + Ingredient format — two cases: + "eggs" → single word: search "eggs", keep ALL results + "cabbage, red" → word + descriptor: search "cabbage, red" (full phrase), + keep only products whose name contains "red" + + 3. For each ingredient: + a. Parse into (search_term, filter_word | None) + - search_term = full ingredient string (e.g. "cabbage, red") + - filter_word = text after the comma (e.g. "red"), or None + b. Open Instacart cross-retailer search: instacart.com/store/s?k= + c. Expand every store's product carousel (Phase 1: Discovery) + d. Click each card, open the detail dialog; extract all fields in one JS call + (Phase 2: Scraping). Fields per product: + store, name, price, price_unit, quantity, image_url, description, out_of_stock + e. If filter_word is set, drop products whose name does not contain it + (case-insensitive substring match) + + 4. Upsert all collected products to the Supabase `ingredients` table. + Columns: id, taxonomy, store, name, price, price_unit, quantity, + image_url, description, out_of_stock + + Single Playwright browser session is reused across all ingredients so that + Instacart's session/location cookies persist between queries. + +Usage +----- + python scraper.py + +Output +------ + Supabase table: ingredients +""" + +import argparse +import asyncio +import os +import re +import uuid +from pathlib import Path +from urllib.parse import quote_plus + +from dotenv import load_dotenv +from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError +from supabase import create_client + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +_ENV_PATH = Path(__file__).parent.parent / ".env" +load_dotenv(_ENV_PATH) + +SUPABASE_URL = os.getenv("SUPABASE_URL") +SUPABASE_KEY = os.getenv("SUPABASE_SERVICE_KEY") + +BASE_URL = "https://www.instacart.com/store/s?k={query}" + +# How long to wait for the product grid to appear (ms). +GRID_WAIT_MS = 5_000 + + +# --------------------------------------------------------------------------- +# Confirmed Instacart DOM selectors (verified against live DOM) +# --------------------------------------------------------------------------- + +STORE_ROW_SELECTOR = '[data-testid="CrossRetailerResultRowWrapper"]' +ITEM_CARD_SELECTOR = '[data-testid^="item_list_item_"]' +NEXT_PAGE_SELECTOR = '[aria-label="Next page"]' +DETAIL_DIALOG_SELECTOR = '[role="dialog"][aria-label="item details"]' +DETAIL_NAME_SELECTOR = '[aria-label="item details"] h2' +LOGIN_MODAL_SELECTOR = ( + '[role="dialog"] input[type="email"], [role="dialog"] input[type="password"], ' + '[aria-modal="true"] input[type="email"], [aria-modal="true"] input[type="password"]' +) + +# Maximum carousel pages to advance per store row. +MAX_CAROUSEL_PAGES = 10 + +# --------------------------------------------------------------------------- +# Ingredient parsing +# --------------------------------------------------------------------------- + +def parse_ingredient(ingredient: str) -> tuple[str, str | None]: + """ + Parse a taxonomy ingredient string into (search_term, filter_word). + + The full ingredient string is always used as the search term so that + Instacart receives the most specific query possible. + + Examples: + "eggs" → ("eggs", None) + "cabbage, red" → ("cabbage, red", "red") + """ + parts = [p.strip() for p in ingredient.split(",", 1)] + search_term = ingredient # search the full phrase + filter_word = parts[1] if len(parts) > 1 else None + return search_term, filter_word + + +def apply_name_filter(products: list[dict], filter_word: str | None) -> list[dict]: + """Keep only products whose name contains filter_word (case-insensitive).""" + if not filter_word: + return products + fw = filter_word.lower() + return [p for p in products if p["name"] and fw in p["name"].lower()] + +# --------------------------------------------------------------------------- +# Core scraper +# --------------------------------------------------------------------------- + +async def scrape_query(page, query: str) -> list[dict]: + """ + Scrape all product cards for *query* on the Instacart cross-retailer page. + Returns a list of product dicts (fields match CSV_FIELDS minus 'taxonomy'). + Returns an empty list if navigation fails or the grid is not found. + """ + url = BASE_URL.format(query=quote_plus(query)) + + # -- Navigate ---------------------------------------------------------- + try: + response = await page.goto(url, wait_until="domcontentloaded", timeout=30_000) + if response and response.status >= 400: + print(f" [ERROR] HTTP {response.status} for query '{query}'") + return [] + except PlaywrightTimeoutError: + print(f" [ERROR] Navigation timeout for query '{query}'") + return [] + except Exception as exc: + print(f" [ERROR] Navigation error for query '{query}': {exc}") + return [] + + # -- Block detection --------------------------------------------------- + try: + body_text = (await page.inner_text("body")).lower() + except Exception: + body_text = "" + + blocked_patterns = [ + "access denied", "403", "robot", "captcha", + "verify you are human", "unusual traffic", "too many requests", "rate limit", + ] + if any(p in body_text for p in blocked_patterns): + print(f" [WARN] Bot-detection page detected for query '{query}' — skipping") + return [] + + if await page.query_selector(LOGIN_MODAL_SELECTOR): + print(f" [WARN] Login modal detected for query '{query}' — skipping") + return [] + + # -- Wait for store rows ----------------------------------------------- + try: + await page.wait_for_selector(STORE_ROW_SELECTOR, timeout=GRID_WAIT_MS) + except PlaywrightTimeoutError: + print(f" [WARN] No store grid found for query '{query}' after {GRID_WAIT_MS}ms") + return [] + + # -- Discovery + Scraping (merged) — process each carousel page inline -- + # Cards are clicked immediately while their carousel page is active, + # so ElementHandles never become stale from carousel advancement. + store_rows = await page.query_selector_all(STORE_ROW_SELECTOR) + + if not store_rows: + print(f" [WARN] No product cards found for query '{query}'") + return [] + + products = [] + total_card_index = 0 + + for row in store_rows: + try: + store_name = await page.evaluate(""" + (row) => { + const lines = (row.innerText || '').split('\\n') + .map(s => s.trim()).filter(Boolean); + return lines[0] || 'Unknown Store'; + } + """, row) + except Exception: + store_name = "Unknown Store" + + seen_testids: set[str] = set() + + for _ in range(MAX_CAROUSEL_PAGES + 1): + cards_now = await row.query_selector_all(ITEM_CARD_SELECTOR) + + # Scrape each new card on this carousel page immediately (before advancing) + for card in cards_now: + try: + testid = await card.get_attribute("data-testid") or "" + except Exception: + testid = "" + if testid in seen_testids: + continue + seen_testids.add(testid) + + i = total_card_index + total_card_index += 1 + + name_text = None + price_text = None + price_unit = None + quantity = None + image_url = None + description = None + out_of_stock = False + item_error = None + + try: + await card.scroll_into_view_if_needed() + await card.click() + await page.wait_for_selector(DETAIL_DIALOG_SELECTOR, timeout=6_000) + except PlaywrightTimeoutError: + item_error = f"Card {i}: timed out waiting for detail dialog" + except Exception as exc: + item_error = f"Card {i}: click error: {exc}" + + if item_error: + print(f" [WARN] {item_error}") + products.append(_empty_product(store_name)) + continue + + # Extract all fields from the open dialog in a single JS round-trip. + try: + _data = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return null; + + // Name — first h2 in dialog + const nameEl = dialog.querySelector('h2'); + const name = nameEl ? nameEl.textContent.trim() : null; + + // Price header span + sibling unit span + // DOM patterns: + // $1.99 /lb per lb → rate → price_unit + // $3.99 18 ct → pkg → quantity + // $9.07 each → fixed → both null + let priceText = '', unitText = '', isRate = false; + const itemDetails = dialog.querySelector('#item_details'); + for (const span of dialog.querySelectorAll('span')) { + if (itemDetails && itemDetails.contains(span)) continue; + if (span.childElementCount !== 0) continue; + const t = span.textContent.trim(); + if (!/^\\$[\\d.,]+/.test(t)) continue; + const parent = span.parentElement; + if (!parent) continue; + const siblings = Array.from(parent.children) + .filter(el => el.tagName === 'SPAN'); + const idx = siblings.indexOf(span); + unitText = (idx !== -1 && siblings[idx + 1]) + ? siblings[idx + 1].textContent.trim() : ''; + isRate = /\\//.test(t) || /^per\\s/i.test(unitText); + priceText = t; + break; + } + + // Price — screen-reader span or first $X.XX span + let price = null; + const spans = dialog.querySelectorAll('span'); + for (const s of spans) { + if (s.childElementCount !== 0) continue; + const t = s.textContent.trim(); + if (t.startsWith('Current price:')) { + const full = t.replace('Current price:', '').trim(); + const m = full.match(/^(\\$[\\d.,]+)/); + price = m ? m[1] : full; + break; + } + } + if (!price) { + for (const s of spans) { + if (s.childElementCount !== 0) continue; + const t = s.textContent.trim(); + const m = t.match(/^(\\$[\\d.,]+)/); + if (m) { price = m[1]; break; } + } + } + + // Image — .ic-image-zoomer img, fallback img[alt]; null if placeholder + let imageUrl = null; + const zoomer = dialog.querySelector('.ic-image-zoomer img'); + if (zoomer && zoomer.src) imageUrl = zoomer.src; + else { + const fb = dialog.querySelector('img[alt]'); + if (fb) imageUrl = fb.src; + } + if (imageUrl && imageUrl.includes('missing-item')) imageUrl = null; + + // Description —

inside container whose sibling h2 is "Details" + let description = null; + for (const h of dialog.querySelectorAll('h2')) { + if (h.textContent.trim() === 'Details') { + const c = h.closest('[tabindex="-1"]') || h.parentElement; + const p = c ? c.querySelector('p') : null; + if (p) description = p.textContent.trim(); + break; + } + } + + // Out of stock — h2 text "out of stock" + let outOfStock = false; + for (const h of dialog.querySelectorAll('h2')) { + if (h.textContent.trim().toLowerCase() === 'out of stock') { + outOfStock = true; break; + } + } + + return { name, priceText, unitText, isRate, price, + imageUrl, description, outOfStock }; + } + """) + except Exception: + _data = None + + if _data: + name_text = _data.get("name") or None + price_text = _data.get("price") or None + image_url = _data.get("imageUrl") or None + description = _data.get("description") or None + out_of_stock = bool(_data.get("outOfStock")) + + _price_raw = _data.get("priceText", "") + _unit_text = _data.get("unitText", "") + if _data.get("isRate"): + _slash = _price_raw.find("/") + if _slash != -1: + _after = _price_raw[_slash + 1:].strip() + if re.match(r'^\d', _after): + quantity = _after + elif _after and re.match( + r'^(?:lb|lbs|oz|fl\.?\s*oz|g|kg|ml|L|gal)\b', + _after, re.IGNORECASE + ): + price_unit = _after + elif _unit_text: + _norm = re.sub( + r'^(?:per\s+|/)', '', _unit_text, flags=re.IGNORECASE + ).strip() + if _norm and not re.match(r'^\d', _norm): + price_unit = _norm + elif _unit_text and not re.match( + r'^\d*\s*each$', _unit_text, re.IGNORECASE + ): + quantity = _unit_text + + # Regex fallback for quantity: two-pass — prefer count patterns + # (e.g. "12-count", "6 pk") over weight/volume so multi-pack items + # resolve to the pack count rather than the per-unit weight. + if quantity is None and price_unit is None and name_text: + m = re.search( + r'\d+(?:[./]\d+)?[\s-]*(?:ct|count|pk|pack)\b', + name_text, re.IGNORECASE + ) + if not m: + m = re.search( + r'\d+(?:[./]\d+)?[\s-]*' + r'(?:oz|fl\.?\s*oz|lb|lbs|g|kg|ml|L|gal)\b', + name_text, re.IGNORECASE + ) + if m: + quantity = m.group(0).strip() + + # Close dialog before moving to next card + try: + await page.keyboard.press("Escape") + await page.wait_for_selector( + DETAIL_DIALOG_SELECTOR, state="hidden", timeout=1_500 + ) + except Exception: + pass # dialog may have already closed; continue regardless + + product_id = testid.removeprefix("item_list_item_") or str(uuid.uuid4()) + products.append({ + "id": product_id, + "store": store_name, + "name": name_text, + "price": price_text, + "price_unit": price_unit, + "quantity": quantity, + "image_url": image_url, + "description": description, + "out_of_stock": out_of_stock, + }) + + # Advance carousel after all cards on this page are processed + try: + next_btn = await row.query_selector(NEXT_PAGE_SELECTOR) + if not next_btn or not await next_btn.is_visible(): + break + await next_btn.click() + await page.wait_for_timeout(150) + except Exception: + break + + return products + + +def _empty_product(store_name: str) -> dict: + return { + "id": str(uuid.uuid4()), "store": store_name, "name": None, "price": None, + "price_unit": None, "quantity": None, "image_url": None, + "description": None, "out_of_stock": False, + } + +# --------------------------------------------------------------------------- +# Pipeline entry point +# --------------------------------------------------------------------------- + +async def run_pipeline(limit: int | None = None) -> None: + """ + Full pipeline: fetch taxonomy → scrape Instacart → upsert to Supabase. + limit: if set, only process the first N ingredients (for test runs). + """ + if not SUPABASE_URL or not SUPABASE_KEY: + raise RuntimeError( + "SUPABASE_URL and SUPABASE_SERVICE_KEY must be set in .env" + ) + + # -- Fetch taxonomy ---------------------------------------------------- + print("Fetching taxonomy from Supabase...") + supabase = create_client(SUPABASE_URL, SUPABASE_KEY) + response = supabase.table("taxonomy").select("ingredient").execute() + ingredients = [row["ingredient"] for row in response.data] + if limit is not None: + ingredients = ingredients[:limit] + print(f" {len(ingredients)} ingredients loaded (limited to first {limit}).") + else: + print(f" {len(ingredients)} ingredients loaded.") + + total_upserted = 0 + + # -- Scrape ------------------------------------------------------------ + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=False, + args=["--disable-blink-features=AutomationControlled"], + ) + context = await browser.new_context( + user_agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36" + ), + viewport={"width": 1280, "height": 900}, + ) + page = await context.new_page() + + for ingredient in ingredients: + search_term, filter_word = parse_ingredient(ingredient) + print(f"\n[{ingredient}] search='{search_term}' filter={filter_word!r}") + + products = await scrape_query(page, search_term) + products = apply_name_filter(products, filter_word) + + rows = [{"taxonomy": ingredient, **p} for p in products] + + if rows: + supabase.table("ingredients").upsert(rows, on_conflict="id").execute() + total_upserted += len(rows) + + print(f" → {len(rows)} products upserted") + + await browser.close() + + print(f"\nDone. {total_upserted} total rows upserted to Supabase.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Instacart pipeline scraper") + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Only process the first N taxonomy ingredients (e.g. --limit 3 for a test run)", + ) + args = parser.parse_args() + asyncio.run(run_pipeline(limit=args.limit)) diff --git a/PlayWright/test_instacart_scraper.py b/PlayWright/test_instacart_scraper.py new file mode 100644 index 0000000..7c3c1ff --- /dev/null +++ b/PlayWright/test_instacart_scraper.py @@ -0,0 +1,712 @@ +""" +test_instacart_scraper.py +-------------------------- +Diagnostic test script for the Instacart Playwright scraper. +Runs in headful mode against a small set of test ingredients, logs raw output +to test_output/, and reports on common failure points. + +Run directly: + python test_instacart_scraper.py + +Or import and call from a runner: + import asyncio + from test_instacart_scraper import run_diagnostics + asyncio.run(run_diagnostics()) + +Output +------ + test_output/raw_.json — raw scraped data + page diagnostics per query + test_output/summary.json — final summary across all queries + +CONFIRMED SELECTORS (verified against live DOM) +------------------------------------------------ + STORE_ROW_SELECTOR '[data-testid="CrossRetailerResultRowWrapper"]' + ITEM_CARD_SELECTOR '[data-testid^="item_list_item_"]' + NEXT_PAGE_SELECTOR '[aria-label="Next page"]' + DETAIL_DIALOG_SEL '[role="dialog"][aria-label="item details"]' + DETAIL_NAME_SEL '[aria-label="item details"] h2' (first h2 in dialog) + price (in-stock) leaf starting with "Current price:" inside dialog + price (out-of-stock) first leaf matching /^\\$[\\d.,]+/ in dialog header + price_unit set when price is a RATE (variable weight/measure): + "$1.99 /lb" + "per lb" → price_unit="lb", quantity=null + Null for fixed-package items and "each" items + quantity set when price covers a FIXED PACKAGE with a known count: + "$3.99" + "18 ct" → quantity="18 ct", price_unit=null + "$9.07" + "each" → both null (single item, qty in name) + Null for rate-priced and single-each items + Fallback: regex extracted from product name + image_url .ic-image-zoomer img src (fallback: first img[alt] in dialog) + description

inside the container whose sibling

reads "Details" +""" + +import asyncio +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +TEST_INGREDIENTS = [ + "chicken breast", + "olive oil", + "milk", + "eggs", + "pasta", +] + +BASE_URL = "https://www.instacart.com/store/s?k={query}" + +# How long to wait for the product grid to appear (ms). +# Instacart can be slow on first load — increase if you see timeout failures. +GRID_WAIT_MS = 8_000 + +OUTPUT_DIR = Path(__file__).parent / "test_output" + +# One row per store on the cross-retailer search page. +STORE_ROW_SELECTOR = '[data-testid="CrossRetailerResultRowWrapper"]' + +# Confirmed by DOM probe: item cards use a unique testid per item with this prefix. +# Pattern: item_list_item_items_{store_id}-{product_id} +ITEM_CARD_SELECTOR = '[data-testid^="item_list_item_"]' + +# "Next page" button inside each store's product carousel. +NEXT_PAGE_SELECTOR = '[aria-label="Next page"]' + +# Maximum number of carousel pages to advance per store (guards against infinite loops). +MAX_CAROUSEL_PAGES = 10 + +# ⚠ Selectors for the product DETAIL overlay — unverified, needs manual DOM check. +# After clicking a card the URL changes to /products/{id}-{slug}?retailerSlug=... +# and an overlay renders the product info. +# +# Confirmed from inspect-element on the live dialog: +# - The overlay is role="dialog" aria-label="item details" (NOT a page nav) +# - Product name lives in an

inside that dialog (the only h1 on the +# page is a hidden "Carts" dialog title — hence the previous timeouts) +# - Price has no aria-label; the most stable hook is the screen-reader-only +# whose text starts with "Current price:" inside the same dialog +DETAIL_DIALOG_SELECTOR = '[role="dialog"][aria-label="item details"]' +DETAIL_NAME_SELECTOR = '[aria-label="item details"] h2' + +# Selector for a login modal/dialog that is actually blocking the page. +# Checks for a visible dialog containing an email or password input field. +# Avoids false positives from the "Log in" nav link that is always present. +LOGIN_MODAL_SELECTOR = '[role="dialog"] input[type="email"], [role="dialog"] input[type="password"], [aria-modal="true"] input[type="email"], [aria-modal="true"] input[type="password"]' + +# Text patterns for a location interstitial — these are specific enough that +# they won't appear in normal nav/header text. +LOCATION_WALL_PATTERNS = [ + "enter your address", + "enter a zip", + "set your location", + "choose a store", + "select a store", +] +BLOCKED_PATTERNS = [ + "access denied", + "403", + "robot", + "captcha", + "verify you are human", + "unusual traffic", + "too many requests", + "rate limit", +] + + +# --------------------------------------------------------------------------- +# Core diagnostic scrape +# --------------------------------------------------------------------------- + +async def scrape_query(page, query: str) -> dict: + """ + Navigate to the Instacart cross-retailer search page for *query* and collect + diagnostic data. Returns a dict ready for JSON serialisation. + """ + url = BASE_URL.format(query=query.replace(" ", "+")) # produces ?k=chicken+breast + result = { + "query": query, + "url": url, + "timestamp": datetime.utcnow().isoformat() + "Z", + "login_wall": False, + "location_wall": False, + "blocked": False, + "grid_found": False, + "store_count": 0, + "card_count": 0, + "products": [], + "errors": [], + "page_title": "", + "page_text_snippet": "", + } + + # -- Navigate -------------------------------------------------------- + try: + response = await page.goto(url, wait_until="domcontentloaded", timeout=30_000) + if response and response.status >= 400: + result["errors"].append(f"HTTP {response.status} on initial navigation") + result["blocked"] = True + return result + except PlaywrightTimeoutError: + result["errors"].append("Navigation timeout (domcontentloaded)") + return result + except Exception as exc: + result["errors"].append(f"Navigation error: {exc}") + return result + + result["page_title"] = await page.title() + + # -- Check for interstitials before waiting for the grid -------------- + try: + body_text = (await page.inner_text("body")).lower() + except Exception: + body_text = "" + + # Keep a snippet for the log so we can eyeball what loaded. + # 1500 chars gives enough context to see whether search results appear. + result["page_text_snippet"] = body_text[:1500].replace("\n", " ") + + # Login wall: check for a visible modal containing an email/password input, + # not just the nav-bar "Log in" link which is always present on the page. + if await page.query_selector(LOGIN_MODAL_SELECTOR): + result["login_wall"] = True + result["errors"].append("Login modal detected (email/password input visible in dialog)") + + if any(p in body_text for p in LOCATION_WALL_PATTERNS): + result["location_wall"] = True + result["errors"].append("Location/address wall detected on page load") + + if any(p in body_text for p in BLOCKED_PATTERNS): + result["blocked"] = True + result["errors"].append("Bot-detection / access-denied page detected") + + # If already blocked, bail early — no point waiting for a grid. + if result["blocked"]: + return result + + # -- Wait for store rows ---------------------------------------------- + try: + await page.wait_for_selector(STORE_ROW_SELECTOR, timeout=GRID_WAIT_MS) + result["grid_found"] = True + except PlaywrightTimeoutError: + result["errors"].append( + f"Timed out waiting for store rows after {GRID_WAIT_MS}ms." + ) + return result + + # Re-check for interstitials now that the grid has loaded. + try: + body_text = (await page.inner_text("body")).lower() + except Exception: + pass + if await page.query_selector(LOGIN_MODAL_SELECTOR): + result["login_wall"] = True + result["errors"].append("Login modal appeared after grid wait") + if any(p in body_text for p in LOCATION_WALL_PATTERNS): + result["location_wall"] = True + result["errors"].append("Location wall appeared after grid wait") + + # -- Phase 1: Discovery — expand each store carousel, collect cards -- + # We collect (store_name, card_element) pairs up-front so we can iterate + # them cleanly in Phase 2 without re-querying the DOM mid-loop. + store_rows = await page.query_selector_all(STORE_ROW_SELECTOR) + result["store_count"] = len(store_rows) + + card_queue = [] # list of (store_name, card_element) + + for row in store_rows: + # Store name: first non-empty line of the row's visible text. + # This picks up "Stater Bros." / "Walmart" / etc. from the row header. + try: + store_name = await page.evaluate(""" + (row) => { + const lines = (row.innerText || '').split('\\n') + .map(s => s.trim()).filter(Boolean); + return lines[0] || 'Unknown Store'; + } + """, row) + except Exception: + store_name = "Unknown Store" + + # Expand the carousel — keep clicking "Next page" until it disappears + # or we hit MAX_CAROUSEL_PAGES, ensuring we see all items in the row. + for _ in range(MAX_CAROUSEL_PAGES): + try: + next_btn = await row.query_selector(NEXT_PAGE_SELECTOR) + if not next_btn or not await next_btn.is_visible(): + break + await next_btn.click() + await page.wait_for_timeout(400) # let carousel animate + except Exception: + break + + # Collect all card elements now visible in this row. + cards_in_row = await row.query_selector_all(ITEM_CARD_SELECTOR) + for card in cards_in_row: + card_queue.append((store_name, card)) + + result["card_count"] = len(card_queue) + + if result["card_count"] == 0: + result["errors"].append("No product cards found across any store row.") + + # -- Phase 2: Scraping — click each card, open dialog, extract data -- + # scroll_into_view_if_needed() ensures the card is clickable even if the + # carousel left it off-screen after the "Next page" expansions. + for i, (store_name, card) in enumerate(card_queue): + name_text = None + price_text = None + price_unit = None + quantity = None + image_url = None + description = None + out_of_stock = False + detail_html_snippet = None + item_error = None + + try: + await card.scroll_into_view_if_needed() + await card.click() + # Wait for the detail dialog to become visible — faster than + # waiting for the URL, and works regardless of URL behaviour. + await page.wait_for_selector(DETAIL_DIALOG_SELECTOR, timeout=6_000) + except PlaywrightTimeoutError: + item_error = f"Card {i}: timed out waiting for detail dialog after click" + result["errors"].append(item_error) + except Exception as exc: + item_error = f"Card {i}: click/navigation error: {exc}" + result["errors"].append(item_error) + + if item_error is None: + # Name: first h2 scoped to the item-details dialog. + # (Confirmed from inspect element — product name is in an h2, not h1.) + try: + name_el = await page.wait_for_selector(DETAIL_NAME_SELECTOR, timeout=4_000) + name_text = (await name_el.inner_text()).strip() or None + except Exception as exc: + result["errors"].append(f"Card {i}: name read error: {exc}") + + # Price unit vs quantity: both read from the mini-header price container + # (the div above #item_details). Structure: + # $1.99 /lb per lb → rate → price_unit="lb" + # $3.99 18 ct → pkg → quantity="18 ct" + # $9.07 each → fixed → both null + # + # Rate signal: price span contains "/" (e.g. "$1.99 /lb") OR unit span + # starts with "per ". Otherwise it's a package quantity. + # "each" → single fixed-price item, both fields stay null. + try: + _unit_info = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return null; + const itemDetails = dialog.querySelector('#item_details'); + for (const span of dialog.querySelectorAll('span')) { + if (itemDetails && itemDetails.contains(span)) continue; + if (span.childElementCount !== 0) continue; + const priceText = span.textContent.trim(); + if (!/^\\$[\\d.,]+/.test(priceText)) continue; + const parent = span.parentElement; + if (!parent) continue; + const siblings = Array.from(parent.children) + .filter(el => el.tagName === 'SPAN'); + const idx = siblings.indexOf(span); + const unitText = (idx !== -1 && siblings[idx + 1]) + ? siblings[idx + 1].textContent.trim() + : ''; + // Rate: price span has "/" (e.g. "$3.99 /lb") + // OR unit span starts with "per " (e.g. "per lb") + const isRate = /\\//.test(priceText) + || /^per\\s/i.test(unitText); + return { priceText, unitText, isRate }; + } + return null; + } + """) + if _unit_info: + _price_raw = _unit_info.get("priceText", "") + _unit_text = _unit_info.get("unitText", "") + if _unit_info.get("isRate"): + # Rate item: the reliable unit is the text after "/" in the + # price span (e.g. "$3.99 /lb" → "lb"). The sibling span for + # rate items often shows a UI label ("1 each") that is not the + # unit — always prefer the "/" extraction. + _slash = _price_raw.find("/") + if _slash != -1: + _after_slash = _price_raw[_slash + 1:].strip() + if re.match(r'^\d', _after_slash): + quantity = _after_slash # "/5.6 lb" → pkg weight + elif _after_slash and re.match( + r'^(?:lb|lbs|oz|fl\.?\s*oz|g|kg|ml|L|gal)\b', + _after_slash, re.IGNORECASE + ): + price_unit = _after_slash # "/lb" → rate unit + # else: "/pkg (est.)" or other non-unit text → both null + elif _unit_text: + # Fallback: "per lb" style (no "/" in price span) + _norm = re.sub( + r'^(?:per\s+|/)', '', _unit_text, flags=re.IGNORECASE + ).strip() + if _norm and not re.match(r'^\d', _norm): + price_unit = _norm + elif _unit_text and not re.match( + r'^\d*\s*each$', _unit_text, re.IGNORECASE + ): + # Fixed package with a meaningful quantity (e.g. "18 ct", "24 oz"). + # Skip "each" / "1 each" — single fixed-price item, qty in name. + quantity = _unit_text + # "each" / "1 each" → both fields stay null + except Exception: + pass + + # Regex fallback for quantity: extract package size from the product name + # when the mini-header unit span is absent or returned "each". + # Two-pass: prefer count/pack descriptors (e.g. "12-count", "6 pk") over + # weight/volume (e.g. "2.05 oz") so multi-pack Costco items resolve + # to the pack count rather than the per-unit size. + # The separator allows optional whitespace OR a hyphen ("12-count"). + if quantity is None and price_unit is None and name_text: + m = re.search( + r'\d+(?:[./]\d+)?[\s-]*(?:ct|count|pk|pack)\b', + name_text, re.IGNORECASE + ) + if not m: + m = re.search( + r'\d+(?:[./]\d+)?[\s-]*' + r'(?:oz|fl\.?\s*oz|lb|lbs|g|kg|ml|L|gal)\b', + name_text, re.IGNORECASE + ) + if m: + quantity = m.group(0).strip() + + # Image URL: the main product image lives inside .ic-image-zoomer + # (a stable non-obfuscated class). Falls back to the first with + # a non-empty alt attribute if the zoomer is absent. + # Instacart's "missing-item" placeholder is not a real image — null it out. + image_url = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return null; + const zoomer = dialog.querySelector('.ic-image-zoomer img'); + if (zoomer && zoomer.src) return zoomer.src; + const fallback = dialog.querySelector('img[alt]'); + return fallback ? fallback.src : null; + } + """) + if image_url and "missing-item" in image_url: + image_url = None + + # Description: inside the container whose sibling

reads "Details". + # Structure confirmed from inspect element: + #
+ #

Details

+ #

description text

+ #
+ description = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return null; + for (const h of dialog.querySelectorAll('h2')) { + if (h.textContent.trim() === 'Details') { + const container = h.closest('[tabindex="-1"]') || h.parentElement; + const p = container ? container.querySelector('p') : null; + return p ? p.textContent.trim() : null; + } + } + return null; + } + """) + + # Out-of-stock detection: an

inside the dialog reads "Out of stock" + # when the item is unavailable. The "Current price:" span is absent in + # this state, so we fall back to a different price extraction path. + try: + out_of_stock = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return false; + for (const h of dialog.querySelectorAll('h2')) { + if (h.textContent.trim().toLowerCase() === 'out of stock') + return true; + } + return false; + } + """) + except Exception: + pass + + # Price extraction: + # In-stock: look for screen-reader-only leaf starting with + # "Current price:" — e.g. "Current price: $1.99 /lb". + # Out-of-stock: that span is absent; fall back to the first leaf + # in the dialog whose text matches /^$[\d.,]+/ (the visible + # price shown in the dialog header, e.g. "$5.97"). + try: + price_text = await page.evaluate(""" + () => { + const dialog = document.querySelector( + '[role="dialog"][aria-label="item details"]' + ); + if (!dialog) return null; + const spans = dialog.querySelectorAll('span'); + + // Primary: screen-reader-only "Current price:" span (in-stock). + // Extract just the dollar amount (e.g. "$1.99") — the unit + // qualifier is captured separately in price_unit. + for (const s of spans) { + if (s.childElementCount !== 0) continue; + const t = s.textContent.trim(); + if (t.startsWith('Current price:')) { + const full = t.replace('Current price:', '').trim(); + const m = full.match(/^(\\$[\\d.,]+)/); + return m ? m[1] : full; + } + } + + // Fallback: first leaf span that looks like a bare price + // (out-of-stock items show the last known price this way). + for (const s of spans) { + if (s.childElementCount !== 0) continue; + const t = s.textContent.trim(); + const m = t.match(/^(\\$[\\d.,]+)/); + if (m) return m[1]; + } + + return null; + } + """) + except Exception as exc: + result["errors"].append(f"Card {i}: price read error: {exc}") + + # If either field is still missing, grab the detail overlay HTML for inspection. + if name_text is None or price_text is None: + try: + detail_html_snippet = (await page.inner_html("main"))[:1200] + except Exception: + detail_html_snippet = "(could not read detail HTML)" + + # Close the dialog with Escape — much faster than go_back() because + # the dialog is an overlay; the search results page never unloaded. + try: + await page.keyboard.press("Escape") + await page.wait_for_selector( + DETAIL_DIALOG_SELECTOR, state="hidden", timeout=3_000 + ) + except PlaywrightTimeoutError: + result["errors"].append(f"Card {i}: dialog did not close after Escape") + break + except Exception as exc: + result["errors"].append(f"Card {i}: dialog close error: {exc}") + break + + result["products"].append({ + "store": store_name, + "name": name_text, + "price": price_text, + "price_unit": price_unit, + "quantity": quantity, + "image_url": image_url, + "description": description, + "out_of_stock": out_of_stock, + "name_missing": name_text is None, + "price_missing": price_text is None, + "detail_html_snippet": detail_html_snippet, + }) + + return result + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + +async def run_diagnostics( + ingredients: list[str] | None = None, + output_dir: Path | None = None, +) -> dict: + """ + Run diagnostic scrapes for each ingredient and write results to output_dir. + Returns the summary dict (also written to output_dir/summary.json). + + Parameters + ---------- + ingredients : list of query strings (defaults to TEST_INGREDIENTS) + output_dir : directory for output files (defaults to ./test_output/) + """ + queries = ingredients or TEST_INGREDIENTS + out = output_dir or OUTPUT_DIR + out.mkdir(parents=True, exist_ok=True) + + all_results = [] + + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=False, # headful so we can see what's happening + args=["--disable-blink-features=AutomationControlled"], + ) + context = await browser.new_context( + user_agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36" + ), + viewport={"width": 1280, "height": 900}, + ) + + # Single page reused across queries to preserve any session cookies + # set after the first navigation (e.g. location cookie). + page = await context.new_page() + + for query in queries: + print(f"\n{'='*60}") + print(f" Testing query: '{query}'") + print(f"{'='*60}") + + result = await scrape_query(page, query) + all_results.append(result) + + # Write per-query raw output. + safe_name = re.sub(r"[^\w]+", "_", query) + raw_path = out / f"raw_{safe_name}.json" + raw_path.write_text(json.dumps(result, indent=2)) + print(f" -> Saved: {raw_path}") + + # Quick console report. + _print_query_report(result) + + await browser.close() + + summary = _build_summary(queries, all_results) + summary_path = out / "summary.json" + summary_path.write_text(json.dumps(summary, indent=2)) + + _print_final_summary(summary) + print(f"\nFull summary -> {summary_path}") + + return summary + + +# --------------------------------------------------------------------------- +# Reporting helpers +# --------------------------------------------------------------------------- + +def _print_query_report(r: dict) -> None: + print(f" Page title : {r['page_title']!r}") + print(f" Login wall : {r['login_wall']}") + print(f" Location wall : {r['location_wall']}") + print(f" Blocked : {r['blocked']}") + print(f" Grid found : {r['grid_found']}") + print(f" Cards found : {r['card_count']}") + + products = r["products"] + complete = sum(1 for p in products if not p["name_missing"] and not p["price_missing"]) + no_name = sum(1 for p in products if p["name_missing"]) + no_price = sum(1 for p in products if p["price_missing"]) + oos = sum(1 for p in products if p.get("out_of_stock")) + + print(f" Products : {len(products)} total, {complete} complete, " + f"{no_name} missing name, {no_price} missing price, {oos} out of stock") + + if r["errors"]: + print(" Errors:") + for e in r["errors"]: + print(f" ✗ {e}") + else: + print(" Errors : none") + + +def _build_summary(queries: list[str], results: list[dict]) -> dict: + per_query = [] + for r in results: + products = r["products"] + complete = sum( + 1 for p in products + if not p["name_missing"] and not p["price_missing"] + ) + per_query.append({ + "query": r["query"], + "login_wall": r["login_wall"], + "location_wall": r["location_wall"], + "blocked": r["blocked"], + "grid_found": r["grid_found"], + "card_count": r["card_count"], + "complete_count": complete, + "errors": r["errors"], + }) + + any_login = any(r["login_wall"] for r in results) + any_location = any(r["location_wall"] for r in results) + any_blocked = any(r["blocked"] for r in results) + all_grids = all(r["grid_found"] for r in results) + total_cards = sum(r["card_count"] for r in results) + total_complete = sum(q["complete_count"] for q in per_query) + + return { + "run_timestamp": datetime.utcnow().isoformat() + "Z", + "queries_tested": queries, + "selectors_used": { + "item_card": ITEM_CARD_SELECTOR, + "detail_dialog": DETAIL_DIALOG_SELECTOR, + "detail_name": DETAIL_NAME_SELECTOR, + "detail_price": "JS: span[text^='Current price:'] inside dialog", + }, + "flags": { + "any_login_wall": any_login, + "any_location_wall": any_location, + "any_blocked": any_blocked, + "all_grids_found": all_grids, + }, + "totals": { + "total_cards": total_cards, + "total_complete": total_complete, + }, + "per_query": per_query, + } + + +def _print_final_summary(s: dict) -> None: + print("\n" + "="*60) + print(" DIAGNOSTIC SUMMARY") + print("="*60) + flags = s["flags"] + print(f" Login wall encountered : {flags['any_login_wall']}") + print(f" Location wall encountered: {flags['any_location_wall']}") + print(f" Request blocked : {flags['any_blocked']}") + print(f" All grids loaded : {flags['all_grids_found']}") + totals = s["totals"] + print(f" Total product cards : {totals['total_cards']}") + print(f" Complete (name+price) : {totals['total_complete']}") + print() + print(f" {'Query':<20} {'Cards':>6} {'Complete':>8} {'Walls/Blocked'}") + print(f" {'-'*20} {'-'*6} {'-'*8} {'-'*20}") + for q in s["per_query"]: + flags_str = ", ".join(filter(None, [ + "login" if q["login_wall"] else "", + "location" if q["location_wall"] else "", + "blocked" if q["blocked"] else "", + "no-grid" if not q["grid_found"] else "", + ])) or "ok" + print(f" {q['query']:<20} {q['card_count']:>6} {q['complete_count']:>8} {flags_str}") + print("="*60) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + asyncio.run(run_diagnostics()) diff --git a/PlayWright/webScrapeInstaCart.py b/PlayWright/webScrapeInstaCart.py new file mode 100644 index 0000000..32b510e --- /dev/null +++ b/PlayWright/webScrapeInstaCart.py @@ -0,0 +1,109 @@ +from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError +import asyncio, json + +STORE_ROW_SELECTOR = '[data-testid="CrossRetailerResultRowWrapper"]' +ITEM_CARD_SELECTOR = '[data-testid^="item_list_item_"]' +NEXT_PAGE_SELECTOR = '[aria-label="Next page"]' +DETAIL_DIALOG_SEL = '[role="dialog"][aria-label="item details"]' +DETAIL_NAME_SEL = '[aria-label="item details"] h2' +MAX_CAROUSEL_PAGES = 10 + +async def scrape_instacart(query: str): + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context( + user_agent=( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36" + ) + ) + page = await context.new_page() + + await page.goto( + f"https://www.instacart.com/store/s?k={query.replace(' ', '+')}", + wait_until="domcontentloaded", + ) + await page.wait_for_selector(STORE_ROW_SELECTOR, timeout=8_000) + + # Phase 1: collect (store_name, card_element) from all store rows, + # expanding each carousel fully before moving on. + store_rows = await page.query_selector_all(STORE_ROW_SELECTOR) + card_queue = [] + + for row in store_rows: + try: + store_name = await page.evaluate(""" + (row) => { + const lines = (row.innerText || '').split('\\n') + .map(s => s.trim()).filter(Boolean); + return lines[0] || 'Unknown Store'; + } + """, row) + except Exception: + store_name = "Unknown Store" + + for _ in range(MAX_CAROUSEL_PAGES): + try: + btn = await row.query_selector(NEXT_PAGE_SELECTOR) + if not btn or not await btn.is_visible(): + break + await btn.click() + await page.wait_for_timeout(400) + except Exception: + break + + for card in await row.query_selector_all(ITEM_CARD_SELECTOR): + card_queue.append((store_name, card)) + + # Phase 2: click each card, scrape name + price from the detail dialog. + results = [] + for store_name, card in card_queue: + try: + await card.scroll_into_view_if_needed() + await card.click() + await page.wait_for_selector(DETAIL_DIALOG_SEL, timeout=6_000) + except PlaywrightTimeoutError: + continue + + name_text = price_text = None + + try: + name_el = await page.wait_for_selector(DETAIL_NAME_SEL, timeout=4_000) + name_text = (await name_el.inner_text()).strip() or None + except Exception: + pass + + try: + price_text = await page.evaluate(""" + () => { + const spans = document.querySelectorAll( + '[role="dialog"][aria-label="item details"] span' + ); + for (const s of spans) { + if (s.childElementCount !== 0) continue; + const t = s.textContent.trim(); + if (t.startsWith('Current price:')) { + return t.replace('Current price:', '').trim(); + } + } + return null; + } + """) + except Exception: + pass + + results.append({"store": store_name, "name": name_text, "price": price_text}) + + try: + await page.keyboard.press("Escape") + await page.wait_for_selector(DETAIL_DIALOG_SEL, state="hidden", timeout=3_000) + except Exception: + break + + await browser.close() + return results + +if __name__ == "__main__": + results = asyncio.run(scrape_instacart("chicken breast")) + print(json.dumps(results, indent=2))