diff --git a/CLAUDE.md b/CLAUDE.md
index adad14b..b9fc128 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -12,14 +12,19 @@ AmazonSellerScraper/
├── popup/ # UI Layer
│ ├── popup.html # Popup interface (dashboard + settings)
│ ├── popup.css # Popup styling
-│ └── popup.js # UI logic + API key management
+│ ├── popup.js # UI logic
+│ └── ai-key.js # Gemini key field (AI chat settings)
├── scripts/
│ ├── content/
│ │ ├── scraper.js # DOM scraping on Amazon pages
│ │ ├── chatbot.js # Floating AI chatbot widget (Shadow DOM)
│ │ └── offer-fetcher.js # Seller price fetching for spread analysis
│ ├── lib/
-│ │ └── parsers.js # Pure search/offer parsing (global Parsers)
+│ │ ├── parsers.js # Pure search/offer parsing (global Parsers)
+│ │ ├── run.js # Scrape run record, bound to one tab (global Run)
+│ │ ├── flags.js # Build flags (global Flags); CLOUD_SYNC is off in 2.1
+│ │ ├── migrate.js # Storage schema migrations, run by the SW
+│ │ └── chat.js # Gemini request builder, run scoping, error text
│ ├── background/
│ │ └── service-worker.js # Message routing + Gemini API calls
│ └── modules/
@@ -69,27 +74,29 @@ AmazonSellerScraper/
### Scraping
1. User clicks "Start Scraping" in popup
-2. `popup.js` sends `START_SCRAPING` via Chrome runtime
-3. `scraper.js` extracts products from Amazon page
-4. Results stored in `chrome.storage.local` via `storage.js`
-5. Auto-navigates to next page (2s delay) until complete
-6. `analyzer.js` generates insights and opportunity scores
-7. User exports via `exporter.js` (Excel/CSV/JSON)
+2. `popup.js` pings the tab; with no answer it offers `chrome.tabs.reload` and starts after the reload
+3. `popup.js` writes a run record (`scripts/lib/run.js`, key `run`) bound to the tab id, then sends `START_SCRAPING`
+4. `scraper.js` classifies the page (results, last, empty, captcha, interstitial, signin, unknown) and extracts products
+5. Results and run progress stored in `chrome.storage.local` in one write per page
+6. Follows the page's Next link after 2 to 4 seconds, up to `settings.maxPages`; on each load the content script asks the SW for its tab id (`WHO_AM_I`) and acts only if its tab owns the run
+7. The run ends with a reason; Stop goes to the run's tab and cancels the pending navigation
+8. `analyzer.js` generates insights and opportunity scores
+9. User exports via `exporter.js` (Excel/CSV/JSON)
### AI Chatbot (client-side, no server)
1. `chatbot.js` injects a floating widget (bottom-right) on Amazon pages with product listings
2. Widget uses Shadow DOM to isolate styles from Amazon's CSS
3. User types a question (e.g. "What's the best deal under $30?")
-4. `chatbot.js` reads scraped products from `chrome.storage.local`
-5. Sends `CHAT_MESSAGE` to `service-worker.js` with question + product data
-6. Service worker calls Gemini API (`gemini-2.0-flash`, free tier) with product context
-7. Response displayed in chat bubble
+4. `chatbot.js` sends `CHAT_MESSAGE` to `service-worker.js` with the question and the last few turns
+5. The service worker reads the user's key and the current run from `chrome.storage.local`; the content script never sees the key
+6. `scripts/lib/chat.js` builds the request: model id in `GEMINI_MODEL`, key in the `x-goog-api-key` header, titles in a fenced JSON block marked untrusted
+7. Response displayed in chat bubble as text, never HTML
## Setup
1. Load unpacked extension in `chrome://extensions`
-2. Click the ProScan popup → open Settings → paste your Gemini API key (free at [aistudio.google.com/apikey](https://aistudio.google.com/apikey))
+2. Click the ProScan popup, expand AI chat settings, paste your Gemini API key (free at [aistudio.google.com/apikey](https://aistudio.google.com/apikey))
3. Navigate to Amazon seller/search page → scrape → export
4. The AI chatbot button appears in the bottom-right corner on Amazon pages with product listings
@@ -109,7 +116,7 @@ npm run test:coverage # With coverage report
- Content scripts (`scraper.js`, `offer-fetcher.js`) have no `module.exports` — loaded via `vm.runInContext` into a JSDOM context with Chrome API mocks and an `innerText` polyfill
- HTML fixtures in `tests/fixtures/` match the exact CSS selectors the code uses
- Chrome APIs (`storage`, `runtime`, `tabs`, `downloads`) are mocked in `tests/setup/chrome-mock.js`
-- XLSX is mocked with jest.fn() stubs for workbook creation
+- XLSX is mocked with jest.fn() stubs in exporter.test.js; export-xlsx.test.js uses the real libs/xlsx.full.min.js
## Chrome APIs Used
@@ -118,34 +125,78 @@ npm run test:coverage # With coverage report
- `chrome.downloads` — file downloads
- `chrome.tabs` — active tab messaging
-## DOM Selectors (Amazon-specific, updated Feb 2026)
+## DOM Selectors (Amazon-specific, updated Sep 2026)
```javascript
-// Product container
+// Product container: real result cards when the page marks them,
+// otherwise any result item with an ASIN
+'[data-component-type="s-search-result"]'
'.s-result-item[data-asin]:not([data-asin=""])'
-// Title — structural first, class fallback
+// Title: the h2 inside the product link, then the last title-recipe h2,
+// then the old fallbacks (a brand line can be its own h2)
+'a h2'
+'[data-cy="title-recipe"] h2'
'h2 span'
'.a-size-base-plus.a-color-base.a-text-normal'
-// Price — data attribute for main price
+// Price: the main price; a-text-price is a unit or list price
'.a-price[data-a-size="xl"] .a-offscreen'
-'.a-price .a-offscreen'
+'.a-price:not(.a-text-price) .a-offscreen' // outside secondary-offer-recipe
-// Rating — cascading: data-cy > star-mini > star-small > plain text
+// Rating: cascading data-cy > star-mini > star-small > plain text
'[data-cy="reviews-ratings-slot"] .a-icon-alt'
'.a-icon-star-mini .a-icon-alt'
'.a-icon-star-small .a-icon-alt'
'[data-cy="reviews-block"] span.a-size-base.a-color-secondary'
-// Review count — aria-label has full number, display text has K/M suffix
-'a[aria-label$="ratings"]'
+// Review count: aria-label has the full number, display text has K/M
+'a[aria-label$="ratings"], a[aria-label$="rating"]'
'.a-size-mini.puis-normal-weight-text.s-underline-text'
+// Sponsored: AdHolder card, the label, or an sspa ad link
+'.puis-sponsored-label-text, [data-component-type="sp-sponsored-result"], a[href*="/sspa/"]'
+
// Prime badge
'.a-icon-prime'
```
+## Product record
+
+One record per ASIN per run, from `Parsers.parseSearchPage` and the scraper:
+
+- `name`, `price`, `rating`, `reviewCount`: null when the card has none, never 0 or "N/A"
+- `priceCents`: integer cents from a dollar price only; `currency` is `USD`, another symbol, or null
+- `url`: always `https://www.amazon.com/dp/{asin}`, never the sspa ad link
+- `sponsored`: true if any placement was an ad; `organicRank`: run-wide rank of the first organic card, or null
+- `placements`: every card the ASIN had, as `{page, position, sponsored, rank}`
+- `delta`: taken once, on the ASIN's first sighting in the run, against `lastValues` from earlier runs.
+ A field that fails to parse keeps its last good value in `lastValues`, with the time in `carried`
+
+## Storage
+
+- `chrome.storage.local` carries `schemaVersion` (3). Storage without it came from 2.0.
+ `scripts/lib/migrate.js` brings it up to date on install, update and browser start;
+ each step is idempotent. 2 to 3 adds `priceCents`, nulls and `/dp/` URLs to 2.0 rows,
+ seeds `lastValues` from them (with `firstSeenAt`), ends a run the update cut off as
+ `updated`, and drops the untouched 2.0 default settings.
+ `tests/fixtures/v2.0-storage.json` is what the live 2.0 build stores, captured with
+ `node tests/e2e/capture-v20-storage.mjs`.
+- `lastValues` is capped by `Delta.prune`: 5,000 ASINs, none older than a year.
+- `Storage` rejects when `chrome.runtime.lastError` is set (code `storage_full` on a
+ quota error). The popup warns from 90% of the quota.
+- Cloud sync is off in 2.1 (`Flags.CLOUD_SYNC`): nothing goes into `syncQueue`, the SW
+ refuses `PROSCAN_EXPORT`, and the popup hides sign-in and Export to ProScan.
+
+## Export
+
+- CSV: UTF-8 with a BOM, CRLF, every text field quoted (RFC 4180). Text starting with
+ `=`, `+`, `-`, `@`, tab or CR gets a leading `'`. Price is a number of dollars from
+ `priceCents`; unknown values are empty cells and real zeros stay 0.
+- Excel: prices, ratings, review counts and scores are numbers; unknowns are empty.
+ `tests/unit/export-xlsx.test.js` writes the workbook with the real SheetJS and checks
+ every XML part parses.
+
## Analytics
### Opportunity Score
diff --git a/README.md b/README.md
index 7a1dc44..2339ef9 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@
-
+
@@ -28,7 +28,7 @@ ProScan is a Chrome extension that scrapes Amazon product listings across multip
- **Multi-page scraping** -- Automatically navigates and extracts product data (name, ASIN, price, rating, reviews, Prime status) across paginated Amazon results
- **Opportunity scoring** -- Proprietary formula identifies high-value arbitrage opportunities based on rating, review velocity, and price positioning
- **Price spread analysis** -- Fetches competing seller prices for each product and calculates variability (Coefficient of Variation) to identify pricing disagreement -- a strong arbitrage signal
-- **AI chatbot** -- Floating widget on Amazon pages answers questions about scraped products using Gemini 2.0 Flash (e.g., "What's the best deal under $30?")
+- **AI chatbot** -- Floating widget on Amazon pages answers questions about your last scan using Gemini Flash and your own free API key (e.g., "What's the best deal under $30?")
- **Analytics dashboard** -- Real-time stats, underpriced product detection, and quality distribution analysis
- **Multi-format export** -- Excel (with styled sheets and charts), CSV, and JSON with full analytics
- **Shadow DOM isolation** -- Chatbot widget styles are fully isolated from Amazon's CSS
@@ -58,11 +58,15 @@ ProScan is a Chrome extension that scrapes Amazon product listings across multip
```
User clicks "Start Scraping"
- → popup.js sends START_SCRAPING message
- → scraper.js extracts products from DOM using cascading selectors
- → Results stored in chrome.storage.local
- → Auto-navigates to next page (2s delay for rate limiting)
- → Repeats until no more pages
+ → popup.js pings the tab (offers a reload if ProScan is not loaded there)
+ → popup.js creates a run bound to that tab and sends START_SCRAPING
+ → scraper.js classifies the page, then extracts products
+ → Results and run progress stored in chrome.storage.local
+ → Follows the page's Next link after a 2 to 4 second delay
+ → Repeats until the last page or the page cap (settings.maxPages, default 20)
+ → The run ends with a reason: complete, stopped, blocked (captcha,
+ bot check, sign-in), selectors_broken, storage_full, interrupted or
+ updated (the extension updated mid-run)
→ analyzer.js generates insights and opportunity scores
→ User exports via exporter.js (Excel/CSV/JSON)
```
@@ -71,10 +75,10 @@ User clicks "Start Scraping"
```
User types question in floating widget
- → chatbot.js reads products from chrome.storage.local
- → Sends CHAT_MESSAGE to service-worker.js
- → Service worker calls Gemini 2.0 Flash API with product context
- → Response displayed in chat bubble
+ → chatbot.js sends CHAT_MESSAGE (question + last few turns) to service-worker.js
+ → Service worker reads the user's key and the current run from storage
+ → scripts/lib/chat.js builds the Gemini request (model id is GEMINI_MODEL there)
+ → Response displayed in chat bubble as plain text
```
## Analytics Engine
@@ -136,6 +140,8 @@ See [docs/PRICE_SPREAD_ANALYSIS.md](docs/PRICE_SPREAD_ANALYSIS.md) for the full
3. Enable **Developer mode** (top-right toggle)
4. Click **Load unpacked** and select the `dist/` folder. The repo root does not load on its own, because the service worker has to be bundled.
+To use the AI chat, open the ProScan popup, expand **AI chat settings** and paste a Gemini API key (free at [aistudio.google.com/apikey](https://aistudio.google.com/apikey)). The key stays in `chrome.storage.local`; only the service worker reads it and sends it to Google.
+
For local Firebase work, `npm run build:dev` points the build at the emulators under the `demo-proscan` project.
## Release checks
@@ -144,6 +150,8 @@ For local Firebase work, `npm run build:dev` points the build at the emulators u
The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, see its README). `npm run test:e2e` loads the built extension into Chromium and runs scrape scenarios against those pages, with every request answered locally. Run `npx playwright install --no-shell chromium` once first. Known bugs run as expected failures tagged with their audit finding id; `PROSCAN_SHOW_KNOWN=1 npm run test:e2e` shows what they fail on.
+The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, see its README). `npm run test:e2e` loads the built extension into Chromium and runs scrape scenarios against those pages, with every request answered locally. Run `npx playwright install --no-shell chromium` once first. Known bugs run as expected failures tagged with their audit finding id; `PROSCAN_SHOW_KNOWN=1 npm run test:e2e` shows what they fail on.
+
## Usage
1. Navigate to any Amazon search results or seller page
@@ -161,7 +169,7 @@ The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, s
| Extension | Chrome Manifest V3 | Extension framework |
| Extension | Shadow DOM | Chatbot style isolation |
| Extension | XLSX.js | Excel generation |
-| AI | Google Gemini 2.0 Flash | Chatbot |
+| AI | Google Gemini Flash (bring your own key) | Chatbot |
## Chrome APIs Used
@@ -185,7 +193,11 @@ AmazonSellerScraper/
│ │ ├── chatbot.js # Floating AI chatbot (Shadow DOM)
│ │ └── offer-fetcher.js # Seller offer page fetching for spread analysis
│ ├── lib/
-│ │ └── parsers.js # Pure search and offer page parsing
+│ │ ├── parsers.js # Pure search and offer page parsing
+│ │ ├── run.js # The scrape run record and its end reasons
+│ │ ├── flags.js # Build flags (cloud sync is off in 2.1)
+│ │ ├── migrate.js # Storage schema migrations (schemaVersion)
+│ │ └── chat.js # Gemini request builder and run scoping
│ ├── background/
│ │ └── service-worker.js # Message routing + Gemini API
│ └── modules/
diff --git a/jest.config.js b/jest.config.js
index fa9d110..765efae 100644
--- a/jest.config.js
+++ b/jest.config.js
@@ -3,12 +3,12 @@ module.exports = {
roots: ['/tests'],
setupFiles: ['/tests/setup/chrome-mock.js'],
testMatch: ['**/*.test.js'],
- // sync.js is the one source file authored as ESM (it is bundled into the
- // service worker by esbuild). A tiny scoped transform rewrites its
- // import/export to CommonJS so it can be unit-tested; every other file
- // keeps the default babel-jest transform.
+ // sync.js and service-worker.js are authored as ESM (esbuild bundles
+ // them). A tiny scoped transform rewrites their import/export to
+ // CommonJS so they can be unit-tested; every other file keeps the
+ // default babel-jest transform.
transform: {
- '[\\\\/]scripts[\\\\/]background[\\\\/]sync\\.js$': '/tests/setup/esm-to-cjs-transform.js',
+ '[\\\\/]scripts[\\\\/]background[\\\\/](sync|service-worker)\\.js$': '/tests/setup/esm-to-cjs-transform.js',
'\\.[jt]sx?$': 'babel-jest'
},
coverageDirectory: 'coverage',
diff --git a/manifest.json b/manifest.json
index 8424448..b194f18 100644
--- a/manifest.json
+++ b/manifest.json
@@ -30,6 +30,8 @@
"js": [
"scripts/modules/price.js",
"scripts/lib/parsers.js",
+ "scripts/lib/run.js",
+ "scripts/lib/flags.js",
"scripts/modules/delta.js",
"scripts/content/scraper.js",
"scripts/content/chatbot.js",
diff --git a/popup/ai-key.js b/popup/ai-key.js
new file mode 100644
index 0000000..0cecc56
--- /dev/null
+++ b/popup/ai-key.js
@@ -0,0 +1,75 @@
+/**
+ * @fileoverview "AI chat settings" in the popup: the user's own Gemini key.
+ *
+ * The key is kept in chrome.storage.local under geminiApiKey. Only the
+ * popup writes it and only the service worker reads it; the chat widget on
+ * Amazon asks the worker whether a key is set and never sees the value.
+ *
+ * @module AiKey
+ */
+
+const AiKey = (() => {
+ const STORAGE_KEY = 'geminiApiKey';
+
+ /** Trims a pasted key. Returns '' for anything that cannot be a key. */
+ function normalize(raw) {
+ const key = String(raw == null ? '' : raw).trim();
+ return /^[A-Za-z0-9_.\-]{20,200}$/.test(key) ? key : '';
+ }
+
+ async function hasKey() {
+ const data = await chrome.storage.local.get([STORAGE_KEY]);
+ return typeof data[STORAGE_KEY] === 'string' && data[STORAGE_KEY].trim() !== '';
+ }
+
+ async function save(raw) {
+ const key = normalize(raw);
+ if (!key) return false;
+ await chrome.storage.local.set({ [STORAGE_KEY]: key });
+ return true;
+ }
+
+ async function clear() {
+ await chrome.storage.local.remove(STORAGE_KEY);
+ }
+
+ function init(doc) {
+ const input = doc.getElementById('geminiKeyInput');
+ const saveBtn = doc.getElementById('geminiKeySave');
+ const clearBtn = doc.getElementById('geminiKeyClear');
+ const status = doc.getElementById('geminiKeyStatus');
+ if (!input || !saveBtn || !clearBtn || !status) return Promise.resolve();
+
+ const show = (set, message) => {
+ status.textContent = message || (set ? 'Key saved. The chat button on Amazon is ready.' : 'No key set. AI chat is off.');
+ input.placeholder = set ? 'Key saved (hidden). Paste a new one to replace it.' : 'Paste your Gemini API key';
+ clearBtn.classList.toggle('hidden', !set);
+ };
+
+ saveBtn.addEventListener('click', async () => {
+ if (await save(input.value)) {
+ input.value = '';
+ show(true);
+ } else {
+ show(await hasKey(), 'That does not look like a Gemini API key.');
+ }
+ });
+ input.addEventListener('keydown', (e) => {
+ if (e.key === 'Enter') saveBtn.click();
+ });
+ clearBtn.addEventListener('click', async () => {
+ await clear();
+ show(false, 'Key removed.');
+ });
+
+ return hasKey().then((set) => show(set));
+ }
+
+ return { STORAGE_KEY, normalize, hasKey, save, clear, init };
+})();
+
+if (typeof module !== 'undefined' && module.exports) {
+ module.exports = AiKey;
+} else {
+ document.addEventListener('DOMContentLoaded', () => AiKey.init(document));
+}
diff --git a/popup/popup.css b/popup/popup.css
index be4ca85..4933826 100644
--- a/popup/popup.css
+++ b/popup/popup.css
@@ -460,6 +460,38 @@ h1 {
color: var(--color-text-primary);
}
+/* AI chat settings */
+.ai-settings summary {
+ cursor: pointer;
+ list-style: none;
+ margin-bottom: 0;
+}
+
+.ai-settings[open] summary {
+ margin-bottom: var(--spacing-sm);
+}
+
+.ai-key-actions {
+ display: flex;
+ gap: var(--spacing-sm);
+ align-items: stretch;
+}
+
+.ai-key-actions .auth-signout {
+ margin-top: var(--spacing-xs);
+}
+
+.ai-key-status,
+.ai-key-help {
+ margin-top: var(--spacing-sm);
+ font-size: var(--font-size-sm);
+ color: var(--color-text-muted);
+}
+
+.ai-key-help a {
+ color: var(--color-accent-green);
+}
+
/* Cloud Export */
.export-cloud-button {
width: 100%;
diff --git a/popup/popup.html b/popup/popup.html
index dc8622c..9e800b8 100644
--- a/popup/popup.html
+++ b/popup/popup.html
@@ -6,6 +6,8 @@
+
+
@@ -43,7 +45,7 @@
ProScan
-
+
@@ -69,10 +71,33 @@
ProScan
+
+
+
+ AI chat settings
+
+
+
+
+
+
+
+
+ Get a free key at aistudio.google.com/apikey.
+ It stays on this computer and is sent only to Google.
+