diff --git a/CLAUDE.md b/CLAUDE.md index 62ede2c..eb5a535 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,11 +4,11 @@ Chrome extension (Manifest V3) that scrapes Amazon seller product listings, provides analytics for resellers/arbitrage, and includes a floating AI chatbot on Amazon pages powered by Gemini API. No server required — everything runs client-side. -## Architecture (v2.2) +## Architecture (v2.3) ```text AmazonSellerScraper/ -├── manifest.json # Extension config (v2.2) +├── manifest.json # Extension config (v2.3) ├── popup/ # UI Layer │ ├── popup.html # Popup interface (dashboard + settings) │ ├── popup.css # Popup styling @@ -23,19 +23,23 @@ AmazonSellerScraper/ │ │ ├── parsers.js # Pure search/offer parsing (global Parsers) │ │ ├── messages.js # Every message type and who may send it (global Msg) │ │ ├── run.js # Run state machine, bound to one tab (global Run) -│ │ ├── flags.js # Build flags (global Flags); CLOUD_SYNC is off until 2.3 +│ │ ├── flags.js # Build flags (global Flags); CLOUD_SYNC is on from 2.3 │ │ ├── migrate.js # Storage schema migrations, run by the SW │ │ └── chat.js # Gemini request builder, run scoping, error text │ ├── background/ │ │ ├── service-worker.js # Wires router, engine, chat, auth and migrations │ │ ├── router.js # The one typed message router │ │ ├── engine.js # The run engine; the only writer of run data -│ │ └── db.js # IndexedDB: runs, products, placements, lastValues, outbox, spread +│ │ ├── db.js # IndexedDB: runs, products, placements, lastValues, outbox, spread +│ │ ├── sync-plan.js # Outbox entry -> cloud writes (pure) +│ │ └── sync.js # Drains the outbox into Firestore, one entry at a time │ └── modules/ │ ├── storage.js # Chrome storage wrapper │ ├── analyzer.js # Data analysis & insights │ ├── spread-analyzer.js # Price spread & arbitrage scoring │ └── exporter.js # Excel/CSV/JSON export +├── packages/ +│ └── schema/index.js # Cloud schema: types, validators, ids (shared with the dashboard) ├── styles/ │ └── chatbot.css # Chatbot widget styles (loaded into Shadow DOM) ├── tests/ # Test suite (Jest) @@ -102,6 +106,37 @@ Run states: idle, starting, running, stopping, then stopped, blocked, failed or `reason` says why it ended: complete, stopped, blocked, selectors_broken, storage_full, storage_error, interrupted or updated. +### Cloud sync (2.3) + +1. The popup signs in with email and password through the SW (`PROSCAN_SIGN_IN`); sign-up opens + the dashboard, and "Forgot password?" sends `PROSCAN_RESET_PASSWORD` (same answer either way) +2. The SW keeps `account` {uid, email} in `chrome.storage.local` while signed in. The engine + queues `{kind:'page', pageIndex, uid}` per saved page and `{kind:'run', uid}` when a run ends, + only when an account is signed in +3. `lastValues` snapshots carry the uid that took them; another account's snapshot gives no delta +4. `scheduleFlush()` runs a few seconds after PAGE_RESULT, STOP_RUN, a tab change that ended the + run, popup open, sign-in, browser start and update. No alarms. After a failed flush the + automatic ones back off (30 s, doubling to 1 h, `lastSync.failures`); Export does not wait +5. `sync.js` drains the outbox for the signed-in uid, oldest first. `sync-plan.js` turns an entry + into writes; most are `set` with `mergeFields`, each field replaced whole. + A product document is a `merge: true` write instead, so the `sourceIds` arrayUnion keeps earlier + sources; the keys its `latest`, `prev` and `delta` lack are sent as deletes. + The run header and source go on each run's last entry in a flush round. + `firstSeenAt` is written only when a read shows the product document does not exist +6. An entry is deleted after its own commit. Entries of another uid are left alone. An entry + that fails schema validation or a rules cap (`RULE_CAPS` in sync-plan.js) is dropped, since + retrying it would fail the same way. The run's end entry goes to `run.syncUid`, the account its + pages were queued for +7. An expired or invalid token signs out and sets `authNotice: 'expired'`; the popup shows + "Your session expired". A `permission-denied` or `invalid-argument` (the rules refusing a + write) is not a sign-out: the entry is marked `failed` and skipped so the rest go on, and + Export retries it. If every entry of a flush is refused, nothing is marked and the error is + thrown, since the account or the rules are the problem + +Cloud paths and shapes: `packages/schema/index.js` (keep the dashboard's copy identical, bump `SV` +on a shape change). Run id `{sourceId}_{startMs}`, minted once in `engine.start`; page id `p0001`; +`dayKey` is the local date the run started. + ### AI Chatbot (client-side, no server) 1. `chatbot.js` injects a floating widget (bottom-right) on Amazon pages with product listings @@ -134,6 +169,9 @@ npm run test:coverage # With coverage report - Pure logic modules (`analyzer.js`, `spread-analyzer.js`) are tested via `require()` directly - Content scripts (`scraper.js`, `offer-fetcher.js`) have no `module.exports` — loaded via `vm.runInContext` into a JSDOM context with Chrome API mocks and an `innerText` polyfill - `tests/setup/engine-rig.js` runs the real router and engine over fake-indexeddb with the real `scraper.js` in a JSDOM page per tab; `tests/unit/engine.test.js` drives runs through it +- `npm run test:contract` runs the real `sync.js` against the Firebase emulators with the dashboard's + `firestore.rules` (`tools/contract.mjs`, `tests/contract/`); ESM files under `packages/` and + `scripts/background/` load in Jest through `tests/setup/esm-to-cjs-transform.js` - `npm run test:e2e` runs the same scenarios in Chromium (`tests/e2e/`), including the worker stopped via CDP between pages and 2.0 and 2.1 builds updated mid-run - HTML fixtures in `tests/fixtures/` match the exact CSS selectors the code uses - Chrome APIs (`storage`, `runtime`, `tabs`, `downloads`) are mocked in `tests/setup/chrome-mock.js` @@ -141,7 +179,7 @@ npm run test:coverage # With coverage report ## Chrome APIs Used -- `chrome.storage.local`: settings, the Gemini key and `schemaVersion` only +- `chrome.storage.local`: settings, the Gemini key, `schemaVersion`, and `account`, `authNotice`, `lastSync` - `chrome.storage.session`: the live run record (content scripts cannot read it) - IndexedDB: run data, in the extension origin; needs no permission - `chrome.runtime.sendMessage/onMessage`: messages, all in `scripts/lib/messages.js` @@ -196,6 +234,8 @@ One record per ASIN per run, from `Parsers.parseSearchPage` and the scraper: - `url`: always `https://www.amazon.com/dp/{asin}`, never the sspa ad link - `sponsored`: true if any placement was an ad; `organicRank`: run-wide rank of the first organic card, or null - `placements`: every card the ASIN had, as `{page, position, sponsored, rank}` +- `img`: the card image on Amazon's image host, or null +- `prev`: the `lastValues` snapshot the delta was taken against, or null - `delta`: taken once, on the ASIN's first sighting in the run, against `lastValues` from earlier runs. A field that fails to parse keeps its last good value in `lastValues`, with the time in `carried` @@ -210,7 +250,7 @@ One record per ASIN per run, from `Parsers.parseSearchPage` and the scraper: the IndexedDB writes commit before anything is removed. - IndexedDB `proscan` (`scripts/background/db.js`): `runs`, `products` (key `[runId, n]`, n is the order found), `placements` (one record per page), `lastValues` (by ASIN), - `outbox` (pages waiting for sync), `spread` and `meta` (`latestRunId`). The popup shows + `outbox` (`{kind, runId, uid, pageIndex}` entries waiting for sync), `spread` and `meta` (`latestRunId`). The popup shows the latest run's products. `tests/fixtures/v2.0-storage.json` is what the live 2.0 build stores, captured with `node tests/e2e/capture-v20-storage.mjs`. @@ -218,8 +258,8 @@ One record per ASIN per run, from `Parsers.parseSearchPage` and the scraper: (`Delta.MAX_ENTRIES`, `Delta.MAX_AGE_DAYS`). - A page write that fails ends the run as `storage_full` (quota) or `storage_error`. The popup warns from 90% of `navigator.storage.estimate()`. -- Cloud sync is off in 2.1 and 2.2 (`Flags.CLOUD_SYNC`): nothing goes into the outbox, the SW - refuses `PROSCAN_EXPORT`, and the popup hides sign-in and Export to ProScan. +- Cloud sync was off in 2.1 and 2.2 and is on from 2.3 (`Flags.CLOUD_SYNC`). Signed out, nothing + goes into the outbox. ## Export diff --git a/README.md b/README.md index 9bd19dc..49d10c2 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,30 @@ User clicks "Start Scraping" → User exports via exporter.js (Excel/CSV/JSON) ``` +### Cloud Sync + +``` +Signed in to a ProScan account in the popup (email and password) + → Each saved page goes into the IndexedDB outbox, tagged with the account's uid; + the end of the run goes in after its pages + → A few seconds after a page, a run end, a popup open or a browser start, + the worker flushes the outbox (no alarms). After a failed flush the next + automatic one waits 30 s, doubling up to an hour + → sync.js writes one entry at a time: the page chunk, and a product and a + history document per ASIN on the page. The run header and the source go + once per run per flush. latest, prev and delta are replaced whole, never + merged; sourceIds keeps every source + → An entry leaves the outbox only once its own writes commit; a page saved + during a flush goes out before the flush returns + → An entry the rules refuse is set aside, so the entries behind it still sync; + the popup counts them and Export to ProScan tries them again + → Export to ProScan flushes right away +``` + +Write cost: a page of k products is 1 + 2k writes, plus 2 per run per flush. A 20 page run of 48 products a page is about 1,960 writes. Spark allows 20,000 writes a day for the whole project, shared by every user, so about 10 such runs a day fill it (audit F-23). + +The document shapes, ids and validators are in `packages/schema/index.js`, which the dashboard copies. Money is integer cents, unknown values are null (left out of compact points), and every document carries `sv`. A signed-out user queues nothing. If Firebase drops the session, the popup says so and the queued pages wait for the same account to sign in again. + ### AI Chatbot Flow ``` @@ -153,6 +177,8 @@ For local Firebase work, `npm run build:dev` points the build at the emulators u The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, see its README). `npm run test:e2e` loads the built extension into Chromium and runs scrape scenarios against those pages, with every request answered locally. Run `npx playwright install --no-shell chromium` once first. Known bugs run as expected failures tagged with their audit finding id; `PROSCAN_SHOW_KNOWN=1 npm run test:e2e` shows what they fail on. +`npm run test:contract` runs the real sync module against the Firebase emulators with the dashboard's `firestore.rules` (from `PROSCAN_RULES`, or a `web` or `proscan-web` checkout next to this repo): queues of 1, 201 and 600 products, a page added mid-flush, replace semantics across runs, create-only `firstSeenAt`, the write count per run, a product in two sources and an entry the rules refuse. It refuses to start if any of its ports is taken, and only stops the emulator processes it started. It needs the Firebase CLI and Java, and uses the `demo-proscan` project only. + The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, see its README). `npm run test:e2e` loads the built extension into Chromium and runs scrape scenarios against those pages, with every request answered locally. Run `npx playwright install --no-shell chromium` once first. Known bugs run as expected failures tagged with their audit finding id; `PROSCAN_SHOW_KNOWN=1 npm run test:e2e` shows what they fail on. ## Usage @@ -176,7 +202,7 @@ The Jest suite includes a golden corpus of saved Amazon pages (`tests/pages/`, s ## Chrome APIs Used -- `chrome.storage.local` -- Settings and the schema version only +- `chrome.storage.local` -- Settings, the schema version, and the signed-in account (`account`, `authNotice`, `lastSync`) - `chrome.storage.session` -- The live run record, written by the service worker - IndexedDB (the extension's own origin, no permission) -- Runs, products, pages, lastValues and the sync outbox. Starting a run keeps the 10 newest runs and removes older ones, except runs still waiting to sync. - `chrome.runtime.sendMessage` / `onMessage` -- Messages, all listed in `scripts/lib/messages.js` @@ -201,19 +227,23 @@ AmazonSellerScraper/ │ │ ├── parsers.js # Pure search and offer page parsing │ │ ├── messages.js # Every message type and who may send it │ │ ├── run.js # The run state machine and its end reasons -│ │ ├── flags.js # Build flags (cloud sync is off until 2.3) +│ │ ├── flags.js # Build flags (cloud sync is on from 2.3) │ │ ├── migrate.js # Storage schema migrations (schemaVersion) │ │ └── chat.js # Gemini request builder and run scoping │ ├── background/ │ │ ├── service-worker.js # Wires the router, the engine, the chat and migrations │ │ ├── router.js # The one message router │ │ ├── engine.js # Runs scrapes; the only writer of run data -│ │ └── db.js # IndexedDB stores +│ │ ├── db.js # IndexedDB stores +│ │ ├── sync-plan.js # What each outbox entry writes (pure) +│ │ └── sync.js # Drains the outbox into Firestore │ └── modules/ │ ├── storage.js # Chrome storage abstraction layer │ ├── analyzer.js # Analytics engine + opportunity scoring │ ├── spread-analyzer.js # Price spread statistics (CV, arbitrage score) │ └── exporter.js # Multi-format export (Excel/CSV/JSON) +├── packages/ +│ └── schema/index.js # Cloud schema shared with the dashboard ├── styles/ │ └── chatbot.css # Chatbot widget styles (Shadow DOM) ├── libs/ diff --git a/jest.config.js b/jest.config.js index 30665ce..61d2d1f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -6,12 +6,13 @@ module.exports = { // Older builds checked out by the e2e upgrade harness have their own tests. testPathIgnorePatterns: ['/node_modules/', '/tests/e2e/\\.build/'], modulePathIgnorePatterns: ['/tests/e2e/\\.build/'], - // sync.js and service-worker.js are authored as ESM (esbuild bundles - // them). A tiny scoped transform rewrites their import/export to - // CommonJS so they can be unit-tested; every other file keeps the - // default babel-jest transform. + // sync.js, sync-plan.js, service-worker.js and the shared schema are + // authored as ESM (esbuild bundles them). A tiny scoped transform + // rewrites their import/export to CommonJS so they can be unit-tested; + // every other file keeps the default babel-jest transform. transform: { - '[\\\\/]scripts[\\\\/]background[\\\\/](sync|service-worker)\\.js$': '/tests/setup/esm-to-cjs-transform.js', + '[\\\\/]scripts[\\\\/]background[\\\\/](sync|sync-plan|service-worker)\\.js$': '/tests/setup/esm-to-cjs-transform.js', + '[\\\\/]packages[\\\\/]schema[\\\\/]index\\.js$': '/tests/setup/esm-to-cjs-transform.js', '\\.[jt]sx?$': 'babel-jest' }, coverageDirectory: 'coverage', @@ -20,6 +21,7 @@ module.exports = { 'scripts/lib/*.js', 'scripts/content/scraper.js', 'scripts/content/offer-fetcher.js', + 'packages/schema/index.js', '!**/node_modules/**' ] }; diff --git a/manifest.json b/manifest.json index fd66f52..71326e9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "ProScan - Amazon Product Scraper", - "version": "2.2.0", + "version": "2.3.0", "description": "Scrape Amazon seller products with analytics and AI-powered product insights", "permissions": [ "storage", diff --git a/package.json b/package.json index dbad2d1..af2da63 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test:integration": "jest tests/integration --verbose", "test:tools": "node --test --test-concurrency=1 \"tools/tests/*.test.mjs\"", "test:e2e": "playwright test", + "test:contract": "node tools/contract.mjs", "test:coverage": "jest --coverage", "lock": "node tools/permission-lock.mjs && node tools/version-gate.mjs", "scan:secrets": "node tools/secret-scan.mjs", diff --git a/packages/schema/index.js b/packages/schema/index.js new file mode 100644 index 0000000..edb605e --- /dev/null +++ b/packages/schema/index.js @@ -0,0 +1,480 @@ +/** + * @fileoverview The ProScan cloud schema: types, validators and id builders. + * + * One file shared by the extension (which writes) and the dashboard (which + * reads). It has no imports and no Firebase code, so the dashboard can copy + * it as is; keep the copies byte-identical and bump SV on any change to a + * document's shape. + * + * Conventions: + * - Money is integer cents. + * - Unknown is null, never 0. In compact points (latest, prev, history + * days, page items) an unknown value is left out, so read it as + * `point.p ?? null`. The Firestore rules check `p is int` when present. + * - Every document carries `sv`, the schema version it was written with. + * - Times are Firestore Timestamps in the cloud. Builders and validators + * here take anything `timeMs` understands: a number of ms, a Date, or an + * object with toMillis(). + * + * Paths, all under workspaces/{uid}: + * sources/{sourceId} SourceDoc + * runs/{runId} RunDoc + * runs/{runId}/pages/{pageId} PageDoc + * products/{asin} ProductDoc (extension fields; lead, + * tags and verdict are the dashboard's) + * products/{asin}/history/daily HistoryDoc + * + * @module Schema + */ + +/** Schema version written into every document. */ +export const SV = 1; + +/** The one marketplace so far. */ +export const MK = 'US'; + +export const ASIN_RE = /^[A-Z0-9]{10}$/; +export const DAY_KEY_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Page chunks expire this long after the run started (TTL policy on expireAt). */ +export const PAGE_TTL_DAYS = 400; + +/** The rules cap a page chunk's items map at this size. */ +export const MAX_PAGE_ITEMS = 120; + +export const RUN_STATUS = ['active', 'complete', 'stopped', 'dead']; +export const SOURCE_TYPES = ['storefront', 'keyword']; + +/** + * @typedef {Object} Point one observation of a product + * @property {number} [p] price, cents + * @property {number} [r] rating, 0 to 5, one decimal + * @property {number} [v] review count + * @property {0|1} [pr] Prime badge + * @property {number} [rk] organic rank within the run, from 1 + * @property {0|1} [sp] sponsored card (page items only) + */ + +/** + * @typedef {Object} Delta change against the previous observation + * @property {number} [p] price change, cents + * @property {number} [pPct] price change, percent of the previous price + * @property {number} [r] rating change + * @property {number} [v] review count change + * @property {number} [days] days between the two observations + */ + +/** + * @typedef {Object} Source + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + */ + +/** + * @typedef {Object} SourceDoc + * @property {number} sv + * @property {string} sourceId + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + * @property {string} lastRunId + * @property {*} lastScrapedAt + * @property {number} [catalogSize] result count Amazon showed, when known + */ + +/** + * @typedef {Object} RunDoc + * @property {number} sv + * @property {string} runId + * @property {string} sourceId + * @property {Source} source + * @property {string} mk + * @property {string} dayKey local date the run started, YYYY-MM-DD + * @property {*} startedAt + * @property {*} finishedAt null while active + * @property {'active'|'complete'|'stopped'|'dead'} status + * @property {?string} reason how it ended, in the extension's words + * @property {number} pagesDone + * @property {number} maxPages + * @property {?number} pagesPlanned known only once a run completes + * @property {?number} totalResultsOnSerp + * @property {{placements:number, uniqueAsins:number, sponsored:number, priceParseFailures:number, newSeen:number}} counters + */ + +/** + * @typedef {Object} PageDoc + * @property {number} sv + * @property {string} runId + * @property {number} page + * @property {*} scrapedAt + * @property {*} expireAt + * @property {number} count products first seen on this page + * @property {number} placements cards on this page + * @property {string} kind results or last + * @property {Object} items every ASIN on the page + * @property {boolean} [truncated] more than MAX_PAGE_ITEMS ASINs + */ + +/** + * @typedef {Object} ProductDoc + * @property {number} sv + * @property {string} asin + * @property {string} mk + * @property {?string} name + * @property {string} url + * @property {?string} img + * @property {Point & {at:*, runId:string, dayKey:string}} latest + * @property {?(Point & {at:*})} prev + * @property {?Delta} delta + * @property {string[]} sourceIds + * @property {*} [firstSeenAt] set once, when the document is created + * @property {string} [firstRunId] + */ + +/** + * @typedef {Object} HistoryDoc + * @property {number} sv + * @property {string} asin + * @property {Object} d dayKey to point + */ + +// ── Ids ───────────────────────────────────────────────────────── + +/** 32-bit FNV-1a of a string, base 36. */ +export function hash(s) { + let h = 0x811c9dc5; + for (const ch of String(s)) { + const c = ch.codePointAt(0); + h ^= c & 0xff; + h = Math.imul(h, 0x01000193); + if (c > 0xff) { + h ^= c >>> 8; + h = Math.imul(h, 0x01000193); + } + } + return (h >>> 0).toString(36); +} + +export function slugify(s) { + return String(s).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); +} + +const SELLER_RE = /^[A-Z0-9]{6,20}$/; +// Query params that change per visit, not per search. +const NOISE = new Set(['page', 'ref', 'qid', 'xpid', 'crid', 'sprefix', 'dib', 'dib_tag', 'sr', 'ds', 'pd_rd_r', 'pf_rd_r']); + +/** The seller in a search URL: me=, seller=, or p_6:/me: inside rh=. */ +function sellerIn(params) { + const direct = params.get('me') || params.get('seller'); + if (direct) return direct.toUpperCase(); + const m = (params.get('rh') || '').match(/(?:^|,)(?:p_6|me):([A-Za-z0-9]+)/); + return m ? m[1].toUpperCase() : null; +} + +/** + * What an Amazon search URL scans. + * @param {string} url + * @returns {Source} + */ +export function sourceOf(url) { + let params = null; + try { + params = new URL(url).searchParams; + } catch (e) { + return { type: 'keyword', sellerId: null, keyword: null, url: url || null }; + } + const seller = sellerIn(params); + if (seller && SELLER_RE.test(seller)) { + return { type: 'storefront', sellerId: seller, keyword: params.get('k') || null, url }; + } + const keyword = params.get('k') || params.get('field-keywords') || null; + return { type: 'keyword', sellerId: null, keyword: keyword ? keyword.trim() : null, url }; +} + +/** + * The source id: s_{sellerId} for a storefront, k_{slug} for a keyword. + * A keyword with non-ASCII letters, or one too long, keeps what it can of + * the slug and adds a hash, so two keywords never share an id. A search + * with neither gets k_x{hash} of its query. + * @param {Source} source + * @returns {string} + */ +export function sourceIdOf(source) { + if (source && source.type === 'storefront' && source.sellerId) return `s_${source.sellerId}`; + const kw = source && source.keyword ? String(source.keyword).normalize('NFC').trim().toLowerCase() : ''; + if (kw) { + const slug = slugify(kw); + // eslint-disable-next-line no-control-regex + const plain = !/[^\x00-\x7f]/.test(kw) && slug.length > 0 && slug.length <= 80; + return plain ? `k_${slug}` : `k_${slug ? slug.slice(0, 60) + '-' : ''}h${hash(kw)}`; + } + let query = ''; + try { + const u = new URL(source.url); + query = [...u.searchParams.entries()] + .filter(([k]) => !NOISE.has(k)) + .map(([k, v]) => `${k}=${v}`) + .sort() + .join('&'); + query = u.pathname + '?' + query; + } catch (e) { /* no URL */ } + return query ? `k_x${hash(query)}` : 'k_unknown'; +} + +/** Run id: {sourceId}_{startMs}, minted once when the run starts. */ +export function runIdOf(sourceId, startMs) { + return `${sourceId}_${startMs}`; +} + +/** Page chunk id: p0001 for page 1. */ +export function pageIdOf(page) { + return 'p' + String(page).padStart(4, '0'); +} + +/** + * The local date at `ms` as YYYY-MM-DD. `tzOffsetMin` is what + * Date#getTimezoneOffset gives where the scan ran (minutes behind UTC), so + * an evening scan in the US files under its own day. + */ +export function dayKeyOf(ms, tzOffsetMin = 0) { + return new Date(ms - tzOffsetMin * 60000).toISOString().slice(0, 10); +} + +export function expireAtMs(startMs) { + return startMs + PAGE_TTL_DAYS * 86400000; +} + +/** + * The cloud status for an extension run: active while it runs, then + * complete, stopped (by the user or a block page) or dead. + */ +export function runStatusOf(state) { + switch (state) { + case 'starting': + case 'running': + case 'stopping': + return 'active'; + case 'done': + return 'complete'; + case 'stopped': + case 'blocked': + return 'stopped'; + default: + return 'dead'; + } +} + +// ── Values ────────────────────────────────────────────────────── + +/** Milliseconds from a number, Date, ISO string or Timestamp; null otherwise. */ +export function timeMs(v) { + if (typeof v === 'number') return Number.isFinite(v) ? v : null; + if (v instanceof Date) return Number.isNaN(v.getTime()) ? null : v.getTime(); + if (typeof v === 'string') { + const t = Date.parse(v); + return Number.isNaN(t) ? null : t; + } + if (v && typeof v.toMillis === 'function') return v.toMillis(); + return null; +} + +const isInt = (v) => Number.isInteger(v); +const round1 = (n) => Math.round(n * 10) / 10; + +/** + * A compact point from an extension product record, leaving out what is + * unknown. + * @param {{priceCents?:?number, rating?:?number, reviewCount?:?number, isPrime?:boolean, organicRank?:?number}} rec + * @returns {Point} + */ +export function pointOf(rec) { + const pt = {}; + if (isInt(rec.priceCents) && rec.priceCents >= 0) pt.p = rec.priceCents; + if (typeof rec.rating === 'number' && rec.rating > 0 && rec.rating <= 5) pt.r = round1(rec.rating); + if (isInt(rec.reviewCount) && rec.reviewCount >= 0) pt.v = rec.reviewCount; + if (typeof rec.isPrime === 'boolean') pt.pr = rec.isPrime ? 1 : 0; + if (isInt(rec.organicRank) && rec.organicRank > 0) pt.rk = rec.organicRank; + return pt; +} + +/** + * The delta between two points, or null when nothing can be compared. + * @param {Point} now + * @param {?Point} prev + * @param {?number} [days] + * @returns {?Delta} + */ +export function deltaOf(now, prev, days = null) { + if (!prev) return null; + const d = {}; + if (isInt(now.p) && isInt(prev.p)) { + d.p = now.p - prev.p; + if (prev.p > 0) d.pPct = round1((d.p / prev.p) * 100); + } + if (typeof now.r === 'number' && typeof prev.r === 'number') d.r = round1(now.r - prev.r); + if (isInt(now.v) && isInt(prev.v)) d.v = now.v - prev.v; + if (Object.keys(d).length === 0) return null; + if (isInt(days) && days >= 0) d.days = days; + return d; +} + +// ── Validators ────────────────────────────────────────────────── +// Each returns a list of problems; empty means valid. + +const isStr = (v) => typeof v === 'string'; +const isStrOrNull = (v) => v === null || isStr(v); +const isTime = (v) => timeMs(v) !== null && typeof v !== 'string'; + +function checkPoint(pt, where, errs, { sp = false } = {}) { + if (!pt || typeof pt !== 'object') { errs.push(`${where}: not a map`); return; } + for (const [k, v] of Object.entries(pt)) { + if (v === null || v === undefined) errs.push(`${where}.${k}: unknown values are left out, not null`); + } + if ('p' in pt && !(isInt(pt.p) && pt.p >= 0)) errs.push(`${where}.p: cents must be a whole number`); + if ('r' in pt && !(typeof pt.r === 'number' && pt.r >= 0 && pt.r <= 5)) errs.push(`${where}.r: rating out of range`); + if ('v' in pt && !(isInt(pt.v) && pt.v >= 0)) errs.push(`${where}.v: review count must be a whole number`); + if ('pr' in pt && pt.pr !== 0 && pt.pr !== 1) errs.push(`${where}.pr: must be 0 or 1`); + if ('rk' in pt && !(isInt(pt.rk) && pt.rk > 0)) errs.push(`${where}.rk: rank must be 1 or more`); + if ('sp' in pt && (!sp || (pt.sp !== 0 && pt.sp !== 1))) errs.push(`${where}.sp: only page items carry sp, as 0 or 1`); +} + +function checkSv(doc, errs) { + if (!doc || typeof doc !== 'object') { errs.push('not a document'); return false; } + if (doc.sv !== SV) errs.push(`sv: expected ${SV}`); + return true; +} + +/** @returns {string[]} */ +export function validateSource(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.sourceId) || !/^[sk]_/.test(doc.sourceId)) errs.push('sourceId: expected s_ or k_'); + if (!SOURCE_TYPES.includes(doc.type)) errs.push('type: storefront or keyword'); + if (!isStrOrNull(doc.sellerId)) errs.push('sellerId: string or null'); + if (!isStrOrNull(doc.keyword)) errs.push('keyword: string or null'); + if (!isStrOrNull(doc.url)) errs.push('url: string or null'); + if (!isStr(doc.lastRunId)) errs.push('lastRunId: string'); + if (!isTime(doc.lastScrapedAt)) errs.push('lastScrapedAt: time'); + if ('catalogSize' in doc && !(isInt(doc.catalogSize) && doc.catalogSize >= 0)) errs.push('catalogSize: whole number'); + return errs; +} + +/** @returns {string[]} */ +export function validateRun(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isStr(doc.sourceId)) errs.push('sourceId: string'); + if (isStr(doc.runId) && isStr(doc.sourceId) && !doc.runId.startsWith(doc.sourceId + '_')) errs.push('runId: must start with sourceId_'); + if (!doc.source || !SOURCE_TYPES.includes(doc.source.type)) errs.push('source.type: storefront or keyword'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if (!isStr(doc.dayKey) || !DAY_KEY_RE.test(doc.dayKey)) errs.push('dayKey: YYYY-MM-DD'); + if (!isTime(doc.startedAt)) errs.push('startedAt: time'); + if (doc.finishedAt !== null && !isTime(doc.finishedAt)) errs.push('finishedAt: time or null'); + if (!RUN_STATUS.includes(doc.status)) errs.push(`status: one of ${RUN_STATUS.join(', ')}`); + if (doc.status === 'active' && doc.finishedAt !== null) errs.push('finishedAt: null while active'); + if (!isStrOrNull(doc.reason)) errs.push('reason: string or null'); + if (!isInt(doc.pagesDone) || doc.pagesDone < 0) errs.push('pagesDone: whole number'); + if (!isInt(doc.maxPages) || doc.maxPages < 1) errs.push('maxPages: whole number'); + if (doc.pagesPlanned !== null && !isInt(doc.pagesPlanned)) errs.push('pagesPlanned: whole number or null'); + if (doc.totalResultsOnSerp !== null && !isInt(doc.totalResultsOnSerp)) errs.push('totalResultsOnSerp: whole number or null'); + const c = doc.counters || {}; + for (const k of ['placements', 'uniqueAsins', 'sponsored', 'priceParseFailures', 'newSeen']) { + if (!isInt(c[k]) || c[k] < 0) errs.push(`counters.${k}: whole number`); + } + return errs; +} + +/** @returns {string[]} */ +export function validatePage(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isInt(doc.page) || doc.page < 1) errs.push('page: 1 or more'); + if (!isTime(doc.scrapedAt)) errs.push('scrapedAt: time'); + if (!isTime(doc.expireAt)) errs.push('expireAt: time'); + if (!isInt(doc.count) || doc.count < 0) errs.push('count: whole number'); + if (!isInt(doc.placements) || doc.placements < 0) errs.push('placements: whole number'); + if (!doc.items || typeof doc.items !== 'object') { + errs.push('items: map'); + } else { + const keys = Object.keys(doc.items); + if (keys.length > MAX_PAGE_ITEMS) errs.push(`items: at most ${MAX_PAGE_ITEMS}`); + for (const asin of keys) { + if (!ASIN_RE.test(asin)) errs.push(`items.${asin}: not an ASIN`); + checkPoint(doc.items[asin], `items.${asin}`, errs, { sp: true }); + } + } + return errs; +} + +/** @returns {string[]} */ +export function validateProduct(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if ('name' in doc && !isStrOrNull(doc.name)) errs.push('name: string or null'); + if ('img' in doc && doc.img !== null && !(isStr(doc.img) && /^https:\/\//.test(doc.img))) errs.push('img: https URL or null'); + if (!isStr(doc.url)) errs.push('url: string'); + if (!doc.latest) { + errs.push('latest: map'); + } else { + const { at, runId, dayKey, ...pt } = doc.latest; + checkPoint(pt, 'latest', errs); + if (!isTime(at)) errs.push('latest.at: time'); + if (!isStr(runId)) errs.push('latest.runId: string'); + if (!isStr(dayKey) || !DAY_KEY_RE.test(dayKey)) errs.push('latest.dayKey: YYYY-MM-DD'); + } + if (doc.prev !== null && doc.prev !== undefined) { + const { at, ...pt } = doc.prev; + checkPoint(pt, 'prev', errs); + if (at !== undefined && !isTime(at)) errs.push('prev.at: time'); + } + if (doc.delta !== null && doc.delta !== undefined) { + const d = doc.delta; + if ('p' in d && !isInt(d.p)) errs.push('delta.p: cents must be a whole number'); + if ('pPct' in d && typeof d.pPct !== 'number') errs.push('delta.pPct: number'); + if ('r' in d && typeof d.r !== 'number') errs.push('delta.r: number'); + if ('v' in d && !isInt(d.v)) errs.push('delta.v: whole number'); + if ('days' in d && !isInt(d.days)) errs.push('delta.days: whole number'); + for (const [k, v] of Object.entries(d)) if (v === null) errs.push(`delta.${k}: left out, not null`); + } + if ('sourceIds' in doc && !(Array.isArray(doc.sourceIds) && doc.sourceIds.every(isStr))) errs.push('sourceIds: list of strings'); + if ('firstSeenAt' in doc && !isTime(doc.firstSeenAt)) errs.push('firstSeenAt: time'); + return errs; +} + +/** @returns {string[]} */ +export function validateHistory(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (!doc.d || typeof doc.d !== 'object') { + errs.push('d: map'); + } else { + for (const [day, pt] of Object.entries(doc.d)) { + if (!DAY_KEY_RE.test(day)) errs.push(`d.${day}: not a day key`); + checkPoint(pt, `d.${day}`, errs); + } + } + return errs; +} + +const VALIDATORS = { + source: validateSource, + run: validateRun, + page: validatePage, + product: validateProduct, + history: validateHistory +}; + +/** Throws when `doc` is not a valid document of `kind`. */ +export function assertValid(kind, doc) { + const errs = VALIDATORS[kind](doc); + if (errs.length) throw new Error(`invalid ${kind} document: ${errs.join('; ')}`); + return doc; +} diff --git a/popup/popup.css b/popup/popup.css index 4933826..2ef101a 100644 --- a/popup/popup.css +++ b/popup/popup.css @@ -412,6 +412,36 @@ h1 { text-align: center; } +.auth-notice { + margin-bottom: var(--spacing-sm); + font-size: var(--font-size-sm); + color: var(--color-status-warning-text); +} + +.auth-links { + display: flex; + justify-content: space-between; + margin-top: var(--spacing-sm); + font-size: var(--font-size-sm); +} + +.auth-links a, +.auth-link-button { + color: var(--color-accent-green); + background: none; + border: none; + padding: 0; + font: inherit; + cursor: pointer; + text-decoration: underline; +} + +.auth-sync { + margin-top: var(--spacing-sm); + font-size: var(--font-size-sm); + color: var(--color-text-muted); +} + .auth-account { display: flex; align-items: center; @@ -460,6 +490,16 @@ h1 { color: var(--color-text-primary); } +#authPanel { + margin-top: var(--spacing-md); +} + +.auth-help { + font-size: var(--font-size-sm); + opacity: 0.7; + margin-bottom: var(--spacing-sm); +} + /* AI chat settings */ .ai-settings summary { cursor: pointer; diff --git a/popup/popup.html b/popup/popup.html index d906be8..d80473f 100644 --- a/popup/popup.html +++ b/popup/popup.html @@ -45,52 +45,6 @@

ProScan

- - - - -
- - AI chat settings - - -
- - -
-
-
- Get a free key at aistudio.google.com/apikey. - It stays on this computer and is sent only to Google. -
-
-
Ready to start
@@ -146,6 +100,59 @@

ProScan

Export to ProScan + + + + +
+ + AI chat settings + + +
+ + +
+
+
+ Get a free key at aistudio.google.com/apikey. + It stays on this computer and is sent only to Google. +
+
+ diff --git a/popup/popup.js b/popup/popup.js index 2ce64c6..8753165 100644 --- a/popup/popup.js +++ b/popup/popup.js @@ -78,6 +78,10 @@ const elements = { authAccount: document.getElementById('authAccount'), authStatusEmail: document.getElementById('authStatusEmail'), authSignOutBtn: document.getElementById('authSignOutBtn'), + authNotice: document.getElementById('authNotice'), + authResetBtn: document.getElementById('authResetBtn'), + authSignUpLink: document.getElementById('authSignUpLink'), + authSync: document.getElementById('authSync'), exportToProScanBtn: document.getElementById('exportToProScanBtn') }; @@ -463,20 +467,53 @@ function sendToWorker(message) { function showSignedIn(user) { currentProScanUser = user; elements.authStatusEmail.textContent = user.email || user.displayName || 'Signed in'; + document.getElementById('authSummary').textContent = 'Syncing to dashboard'; elements.authForm.classList.add('hidden'); elements.authAccount.classList.remove('hidden'); elements.authError.classList.add('hidden'); elements.authError.textContent = ''; + elements.authNotice.classList.add('hidden'); elements.exportToProScanBtn.classList.toggle('hidden', !Flags.CLOUD_SYNC); } +/** + * The sync line under the account: what is waiting, or when it last synced. + * + * @param {{pending?: number, failed?: number, lastSync?: ?{at:number, error:?string}}} state + */ +function showSyncState(state) { + const pending = (state && state.pending) || 0; + const failed = (state && state.failed) || 0; + const last = state && state.lastSync; + let text = 'Scans sync to your ProScan dashboard automatically.'; + if (pending > 0) { + text = `${pending} page${pending === 1 ? '' : 's'} waiting to sync.`; + if (last && last.error) text += ' The last try failed; it will try again in a while, or now with Export to ProScan.'; + } else if (last && !last.error) { + text = 'Everything is synced.'; + } + if (failed > 0) { + text += ` ${failed} page${failed === 1 ? '' : 's'} could not sync: ProScan refused ${failed === 1 ? 'it' : 'them'}. Export to ProScan tries again.`; + } + elements.authSync.textContent = text; + elements.authSync.classList.remove('hidden'); +} + /** * Render the signed-out state: show the sign-in form, hide the account state, * and hide the cloud-export button. */ -function showSignInForm() { +function showSignInForm(notice = null) { currentProScanUser = null; elements.authAccount.classList.add('hidden'); + elements.authSync.classList.add('hidden'); + elements.authNotice.textContent = notice === 'expired' + ? 'Your session expired. Sign in again to keep syncing; your scans are kept.' + : ''; + elements.authNotice.classList.toggle('hidden', notice !== 'expired'); + document.getElementById('authSummary').textContent = 'Dashboard sync (optional)'; + // An expired session is the one case worth unfolding the panel for. + if (notice === 'expired') document.getElementById('authPanel').open = true; elements.authForm.classList.remove('hidden'); elements.authError.classList.add('hidden'); elements.authError.textContent = ''; @@ -494,6 +531,30 @@ function showAuthError(message) { elements.authError.classList.remove('hidden'); } +/** + * Ask the worker to send a password reset email for the typed address. + * The answer is the same whether or not the account exists. + * + * @async + */ +async function handleResetPassword() { + const email = elements.authEmail.value.trim(); + if (!email) { + showAuthError('Enter your email above, then tap Forgot password.'); + return; + } + elements.authResetBtn.disabled = true; + const response = await sendToWorker({ type: Msg.T.PROSCAN_RESET_PASSWORD, email }); + elements.authResetBtn.disabled = false; + if (response && response.ok) { + elements.authError.classList.add('hidden'); + elements.authNotice.textContent = response.message; + elements.authNotice.classList.remove('hidden'); + } else { + showAuthError((response && response.error) || 'Could not send the reset email.'); + } +} + /** * Restore the sign-in button to its idle state. */ @@ -513,10 +574,12 @@ async function initializeAuthUI() { if (!Flags.CLOUD_SYNC) return; document.getElementById('authPanel').classList.remove('hidden'); const response = await sendToWorker({ type: Msg.T.PROSCAN_AUTH_STATE }); + if (response && response.dashboardUrl) elements.authSignUpLink.href = response.dashboardUrl; if (response && response.user) { showSignedIn(response.user); + showSyncState(response); } else { - showSignInForm(); + showSignInForm(response && response.notice); } } @@ -545,6 +608,7 @@ async function handleSignIn() { if (response && response.user) { elements.authPassword.value = ''; showSignedIn(response.user); + showSyncState({ pending: 0 }); } else { showAuthError((response && response.error) || 'Sign-in failed.'); resetSignInButton(); @@ -591,11 +655,18 @@ async function handleExportToProScan() { if (response && response.ok) { const count = response.products || 0; - if (!response.written || count === 0) { - updateStatus('No new products to export', 'info'); + const failed = response.failed || 0; + if (failed > 0) { + updateStatus(`Synced ${count} product${count === 1 ? '' : 's'}; ${failed} page${failed === 1 ? '' : 's'} could not sync.`, 'warning'); + } else if (!response.entries) { + updateStatus('Everything is already synced.', 'info'); } else { - updateStatus(`Exported ${count} product${count === 1 ? '' : 's'} to ProScan`, 'success'); + updateStatus(`Synced ${count} product${count === 1 ? '' : 's'} to ProScan`, 'success'); } + showSyncState({ pending: 0, failed, lastSync: { at: Date.now(), error: null } }); + } else if (response && response.expired) { + showSignInForm('expired'); + updateStatus(response.error, 'warning'); } else { updateStatus((response && response.error) || 'Export failed.', 'error'); } @@ -610,6 +681,7 @@ async function handleExportToProScan() { // ProScan cloud auth + export elements.authSignInBtn.addEventListener('click', handleSignIn); elements.authSignOutBtn.addEventListener('click', handleSignOut); +elements.authResetBtn.addEventListener('click', handleResetPassword); elements.exportToProScanBtn.addEventListener('click', handleExportToProScan); // Enter-to-submit from the password field diff --git a/scripts/background/engine.js b/scripts/background/engine.js index 329fbfc..f366113 100644 --- a/scripts/background/engine.js +++ b/scripts/background/engine.js @@ -25,6 +25,7 @@ const Run = require('../lib/run.js'); const Msg = require('../lib/messages.js'); const Flags = require('../lib/flags.js'); const Delta = require('../modules/delta.js'); +const Schema = require('../../packages/schema/index.js'); /** A page asked for but not reported this long is asked for again. */ const PAGE_TIMEOUT_MS = 20000; @@ -77,14 +78,26 @@ function durable(run) { return rest; } -function runSuffix(random) { - return Math.floor(random() * 0x100000000).toString(36).padStart(7, '0').slice(0, 8); +/** The signed-in account's uid, as the service worker keeps it in storage.local. */ +async function accountUid(chrome) { + const data = await chrome.storage.local.get('account'); + return (data && data.account && data.account.uid) || null; +} + +/** + * A lastValues snapshot counts for `uid` when this account took it, or when + * it was taken signed out. Another account's snapshot does not. + */ +function prevFor(snap, uid) { + if (!snap) return null; + return !snap.uid || snap.uid === uid ? snap : null; } /** * @param {Object} deps * @param {Object} deps.chrome - chrome.* (storage.session, storage.local, tabs) * @param {function(): Promise} deps.openDb - resolves to the db.js api + * @param {function(): Promise} [deps.owner] - uid of the signed-in account */ function createEngine({ chrome, @@ -94,7 +107,8 @@ function createEngine({ setTimer = setTimeout, clearTimer = clearTimeout, flags = Flags, - log = console + log = console, + owner = () => accountUid(chrome) }) { let dbPromise = null; let timer = null; @@ -152,13 +166,35 @@ function createEngine({ timer = null; } + /** Outbox entries for a signed-in account; none when sync is off or nobody is signed in. */ + async function outbox(runId, kind, extra = {}) { + if (!flags.CLOUD_SYNC) return []; + const uid = await owner(); + if (!uid) return []; + return [{ store: 'outbox', put: { runId, kind, uid, queuedAt: now(), ...extra } }]; + } + + /** + * The end-of-run entry for `run`. It goes to the account its pages were + * queued for (run.syncUid), even if that account signed out mid-run, so + * the cloud header does not stay active forever. + */ + async function endEntry(run) { + if (!flags.CLOUD_SYNC || !(run.page > 0)) return []; + const uid = run.syncUid || await owner(); + if (!uid) return []; + return [{ store: 'outbox', put: { runId: run.runId, kind: 'run', uid, queuedAt: now() } }]; + } + /** Ends `run` for `reason` and tells its tab. Returns the ended run. */ async function end(run, reason) { const ended = Run.finish(run, reason, now()); cancelTimer(); await saveRun(ended); try { - await (await db()).write([{ store: 'runs', put: durable(ended) }]); + // A run with saved pages sends its final header to the cloud. + const queued = await endEntry(ended); + await (await db()).write([{ store: 'runs', put: durable(ended) }, ...queued]); } catch (err) { log.warn('[ProScan] Could not record the end of the run:', err.message); } @@ -212,8 +248,15 @@ function createEngine({ const local = await chrome.storage.local.get('settings'); const settings = (local && local.settings) || {}; const t = now(); - const runId = `${t}-${runSuffix(random)}`; - let run = Run.create({ runId, tabId, maxPages: settings.maxPages, source: Run.sourceOf(pong.url, t), now: t }); + // One id for the run, here and in the cloud: {sourceId}_{startMs}. + const found = Schema.sourceOf(pong.url); + const sourceId = Schema.sourceIdOf(found); + const runId = Schema.runIdOf(sourceId, t); + let run = Run.create({ + runId, tabId, maxPages: settings.maxPages, now: t, + source: { ...found, sourceId, startedAt: new Date(t).toISOString() } + }); + run = { ...run, sourceId, dayKey: Schema.dayKeyOf(t, new Date(t).getTimezoneOffset()) }; await saveRun(run); const first = [ { store: 'runs', put: durable(run) }, @@ -261,7 +304,7 @@ function createEngine({ const rec = latestId ? await store.get('runs', latestId) : null; if (!Run.isActive(rec)) return null; const ended = Run.finish(rec, reason, now()); - await store.write([{ store: 'runs', put: durable(ended) }]); + await store.write([{ store: 'runs', put: durable(ended) }, ...await endEntry(ended)]); return ended; } @@ -320,16 +363,23 @@ function createEngine({ store = await db(); const existing = await store.runProducts(runId); const { fresh, changed } = foldPage(existing, result.products, page); - const prevs = await store.getMany('lastValues', fresh.map(p => p.asin)); + const snaps = await store.getMany('lastValues', fresh.map(p => p.asin)); + const uid = await owner(); ops = []; fresh.forEach((p, i) => { p.runId = runId; p.pageIndex = page; p.n = existing.length + i; - const prev = prevs[i] || null; + const prev = prevFor(snaps[i], uid); // Only an ASIN's first sighting in the run gets a delta. p.delta = Delta.computeDeltas(p, prev); - ops.push({ store: 'lastValues', put: { asin: p.asin, ...Delta.snapshot(p, prev) } }); + p.prev = prev ? { + priceCents: prev.priceCents ?? null, rating: prev.rating ?? null, + reviewCount: prev.reviewCount ?? null, scrapedAt: prev.scrapedAt || null, uid: prev.uid || null + } : null; + const snap = { asin: p.asin, ...Delta.snapshot(p, prev) }; + if (uid) snap.uid = uid; + ops.push({ store: 'lastValues', put: snap }); ops.push({ store: 'products', put: p }); }); changed.forEach(p => ops.push({ store: 'products', put: p })); @@ -337,17 +387,24 @@ function createEngine({ store: 'placements', put: { runId, pageIndex: page, count: fresh.length, placements: result.placements || 0, - kind: result.kind, fill: result.fill || null, scrapedAt: new Date(t).toISOString(), url + kind: result.kind, fill: result.fill || null, scrapedAt: new Date(t).toISOString(), url, + total: Number.isInteger(result.total) && result.total > 0 ? result.total : null } }); - if (flags.CLOUD_SYNC) ops.push({ store: 'outbox', put: { runId, pageIndex: page, queuedAt: t } }); + const queuedPage = await outbox(runId, 'page', { pageIndex: page }); + ops.push(...queuedPage); next = { ...run, page, itemCount: run.itemCount + fresh.length, heartbeat: t, lastUrl: url, nextHref: result.nextHref || null, awaiting: false }; - if (ending) next = Run.finish(next, ending, t); - else next.navAt = t + Run.pageDelay(random()); + if (queuedPage.length && !next.syncUid) next.syncUid = queuedPage[0].put.uid; + if (ending) { + next = Run.finish(next, ending, t); + ops.push(...await endEntry(next)); + } else { + next.navAt = t + Run.pageDelay(random()); + } ops.push({ store: 'runs', put: durable(next) }); await store.write(ops); } catch (err) { @@ -415,10 +472,13 @@ function createEngine({ }).then((out) => { tick(); return out; }); } + /** Resolves true when this ended the run. */ function tabRemoved(tabId) { return serial(async () => { const run = await getRun(); - if (Run.isActive(run) && run.tabId === tabId) await end(run, 'interrupted'); + if (!Run.isActive(run) || run.tabId !== tabId) return false; + await end(run, 'interrupted'); + return true; }); } @@ -428,13 +488,14 @@ function createEngine({ * the tabs permission a non-Amazon URL is hidden, which also counts. */ function tabUpdated(tabId, info, tab) { - if (!info || info.status !== 'loading') return Promise.resolve(); + if (!info || info.status !== 'loading') return Promise.resolve(false); return serial(async () => { const run = await getRun(); - if (!Run.owns(run, tabId) || run.state !== 'running' || run.awaiting) return; + if (!Run.owns(run, tabId) || run.state !== 'running' || run.awaiting) return false; const url = info.url || (tab && tab.url); - if (url && url === run.lastUrl) return; + if (url && url === run.lastUrl) return false; await end(run, 'interrupted'); + return true; }); } @@ -450,7 +511,8 @@ function createEngine({ const latest = await store.getMeta('latestRunId'); const rec = latest ? await store.get('runs', latest) : null; if (Run.isActive(rec)) { - await store.write([{ store: 'runs', put: durable(Run.finish(rec, reason, now())) }]); + const queued = await endEntry(rec); + await store.write([{ store: 'runs', put: durable(Run.finish(rec, reason, now())) }, ...queued]); } }); } @@ -515,34 +577,10 @@ function createEngine({ }; } - /** - * The sync bundle for every run in the outbox, built from the products - * as they are now, and the outbox keys to delete once it is written. - */ - async function outboxBundle() { - const store = await db(); - const entries = await store.getAll('outbox'); - const runIds = [...new Set(entries.map(e => e.runId))]; - const syncQueue = []; - const scrapeRunPages = []; - let scrapeRunMeta = null; - for (const runId of runIds) { - syncQueue.push(...await store.runProducts(runId)); - scrapeRunPages.push(...await store.runPages(runId)); - const rec = await store.get('runs', runId); - if (rec) scrapeRunMeta = rec.source; - } - return { bundle: { syncQueue, scrapeRunMeta, scrapeRunPages }, seqs: entries.map(e => e.seq) }; - } - - async function clearOutbox(seqs) { - await (await db()).write(seqs.map(seq => ({ store: 'outbox', delete: seq }))); - } - return { start, stop, pageReady, pageResult, heartbeat, tick, tabRemoved, tabUpdated, recover, - getState, getResults, spreadResult, chatData, outboxBundle, clearOutbox, db + getState, getResults, spreadResult, chatData, db }; } -module.exports = { createEngine, foldPage, durable, PAGE_TIMEOUT_MS, KEEP_RUNS }; +module.exports = { createEngine, foldPage, durable, prevFor, PAGE_TIMEOUT_MS, KEEP_RUNS }; diff --git a/scripts/background/service-worker.js b/scripts/background/service-worker.js index 13cdecb..936f601 100644 --- a/scripts/background/service-worker.js +++ b/scripts/background/service-worker.js @@ -1,10 +1,11 @@ -import { auth } from './firebase-init.js'; +import { auth, db as firestore } from './firebase-init.js'; import { onAuthStateChanged, signInWithEmailAndPassword, + sendPasswordResetEmail, signOut, } from 'firebase/auth/web-extension'; -import { syncToCloud } from './sync.js'; +import { createSync, isAuthError } from './sync.js'; import { createRouter } from './router.js'; import { createEngine } from './engine.js'; import DB from './db.js'; @@ -39,11 +40,16 @@ const chatDeps = { }; // The run +// Saved pages and ended runs wake the sync; see scheduleFlush below. +const thenFlush = (p) => p.then((out) => { scheduleFlush(); return out; }); +// A tab event wakes it only when it ended the run. +const flushIfEnded = (p) => p.then((ended) => { if (ended) scheduleFlush(); return ended; }); + router.on(Msg.T.START_RUN, (m) => engine.start(m)); -router.on(Msg.T.STOP_RUN, () => engine.stop()); +router.on(Msg.T.STOP_RUN, () => thenFlush(engine.stop())); router.on(Msg.T.GET_STATE, () => engine.getState()); router.on(Msg.T.PAGE_READY, (m, sender) => engine.pageReady(m, sender)); -router.on(Msg.T.PAGE_RESULT, (m, sender) => engine.pageResult(m, sender)); +router.on(Msg.T.PAGE_RESULT, (m, sender) => thenFlush(engine.pageResult(m, sender))); router.on(Msg.T.HEARTBEAT, (m, sender) => engine.heartbeat(m, sender)); // Spread analysis @@ -80,34 +86,58 @@ chrome.runtime.onInstalled.addListener((details) => { }); } else if (details.reason === 'update') { console.log('[ProScan] Extension updated to version', chrome.runtime.getManifest().version); - migrate().then(() => engine.recover('updated')); + migrate().then(() => engine.recover('updated')).then(() => scheduleFlush()); } }); // A run cannot survive a browser restart, since its tab id is gone. chrome.runtime.onStartup.addListener(() => { - migrate().then(() => engine.recover()); + migrate().then(() => engine.recover()).then(() => scheduleFlush()); }); // Neither listener needs the tabs permission. -chrome.tabs.onRemoved.addListener((tabId) => { engine.tabRemoved(tabId); }); -chrome.tabs.onUpdated.addListener((tabId, info, tab) => { engine.tabUpdated(tabId, info, tab); }); +chrome.tabs.onRemoved.addListener((tabId) => { flushIfEnded(engine.tabRemoved(tabId)); }); +chrome.tabs.onUpdated.addListener((tabId, info, tab) => { flushIfEnded(engine.tabUpdated(tabId, info, tab)); }); + +// ── ProScan account and cloud sync ────────────────────────────────────────── +// The popup is a plain page; it signs in and exports through these messages, +// and the worker owns the one Firebase instance. +// +// storage.local keeps `account` {uid, email} while signed in, so the engine +// queues pages for that account, and `authNotice` 'expired' when Firebase +// dropped the session without the user signing out. + +const DASHBOARD_URL = 'https://proscanbot.web.app/dashboard/'; +const FLUSH_DELAY_MS = 3000; +let flushTimer = null; + +/** + * After a failed automatic flush the next one waits 30 s, doubling per + * failure up to an hour, so a refused or throttled entry is not replayed + * (with its product reads) on every page load. Export ignores the wait. + */ +function retryDelayMs(failures) { + if (!failures) return 0; + return Math.min(30000 * 2 ** (failures - 1), 60 * 60 * 1000); +} -// ════════════════════════════════════════════════════════════════════════════ -// M3 cloud sync — Firebase Auth (extension-native) + Firestore write path. -// The popup is a plain (unbundled) page; it drives sign-in / export by sending -// these messages to the worker, which owns the single Firebase instance. -// ════════════════════════════════════════════════════════════════════════════ +const sync = createSync({ db: firestore, openStore: () => engine.db() }); /** Resolve the current Firebase user, waiting for auth to rehydrate from * IndexedDB after a cold service-worker start. */ function currentUser() { return new Promise((resolve) => { if (auth.currentUser) return resolve(auth.currentUser); - const unsub = onAuthStateChanged(auth, (u) => { - unsub(); + let settled = false; + let unsub = null; + unsub = onAuthStateChanged(auth, (u) => { + if (settled) return; + settled = true; + if (unsub) unsub(); resolve(u); }); + // The first answer can come before onAuthStateChanged returns. + if (settled) unsub(); }); } @@ -115,6 +145,73 @@ function currentUser() { const publicUser = (u) => u ? { uid: u.uid, email: u.email, displayName: u.displayName } : null; +let signingOut = false; + +async function rememberAccount(user) { + if (user) { + await chrome.storage.local.set({ account: { uid: user.uid, email: user.email || null } }); + await chrome.storage.local.remove('authNotice'); + return; + } + const { account } = await chrome.storage.local.get('account'); + if (!account) return; + await chrome.storage.local.remove('account'); + // Signed out without asking: the refresh token was revoked or expired. + if (!signingOut) await chrome.storage.local.set({ authNotice: 'expired' }); +} + +onAuthStateChanged(auth, (user) => { + rememberAccount(user).catch(() => {}); + if (user) scheduleFlush(); +}); + +/** Firebase refused the session: sign out and say so in the popup. */ +async function expireSession() { + await chrome.storage.local.set({ authNotice: 'expired' }); + await chrome.storage.local.remove('account'); + await signOut(auth).catch(() => {}); +} + +/** + * Writes the outbox for the signed-in account now. `manual` (Export) also + * retries entries the rules refused and skips the backoff. + */ +async function flushNow({ manual = false } = {}) { + if (!Flags.CLOUD_SYNC) return { skipped: true }; + const user = await currentUser(); + if (!user) return { skipped: true }; + const { lastSync } = await chrome.storage.local.get('lastSync'); + const failures = (lastSync && lastSync.error && lastSync.failures) || 0; + if (!manual && lastSync && lastSync.error && Date.now() < lastSync.at + retryDelayMs(failures)) { + return { skipped: true, backoff: true }; + } + try { + const out = await sync.flush(user.uid, { retryFailed: manual }); + await chrome.storage.local.set({ lastSync: { at: Date.now(), error: null, failures: 0 } }); + return out; + } catch (err) { + await chrome.storage.local.set({ + lastSync: { at: Date.now(), error: (err && err.code) || 'unknown', failures: failures + 1 }, + }); + if (isAuthError(err)) await expireSession(); + throw err; + } +} + +/** + * Flushes a few seconds after the last page or run end, on wake events the + * worker already gets. No alarm: the timer dies with the worker, and what + * it missed goes out on the next wake. + */ +function scheduleFlush(ms = FLUSH_DELAY_MS) { + if (!Flags.CLOUD_SYNC) return; + if (flushTimer) clearTimeout(flushTimer); + flushTimer = setTimeout(() => { + flushTimer = null; + flushNow().catch(() => {}); + }, ms); +} + /** Map Firebase auth error codes to friendly popup messages. */ function friendlyAuthError(err) { switch (err && err.code) { @@ -131,34 +228,70 @@ function friendlyAuthError(err) { case 'auth/operation-not-allowed': return 'Email sign-in is not enabled for this project yet.'; default: - return ((err && err.message) || 'Sign-in failed.').replace(/^Firebase:\s*/, ''); + return 'Sign-in failed. Try again.'; } } -router.on(Msg.T.PROSCAN_AUTH_STATE, async () => ({ user: publicUser(await currentUser()) })); +router.on(Msg.T.PROSCAN_AUTH_STATE, async () => { + const user = await currentUser(); + const { authNotice, lastSync } = await chrome.storage.local.get(['authNotice', 'lastSync']); + // Opening the popup is a wake event too. + if (user) scheduleFlush(0); + return { + user: publicUser(user), + notice: user ? null : authNotice || null, + pending: user ? await sync.pending(user.uid).catch(() => 0) : 0, + failed: user ? await sync.failed(user.uid).catch(() => 0) : 0, + lastSync: lastSync || null, + dashboardUrl: DASHBOARD_URL, + }; +}); router.on(Msg.T.PROSCAN_SIGN_IN, (m) => - signInWithEmailAndPassword(auth, m.email, m.password) - .then((cred) => ({ user: publicUser(cred.user) })) + signInWithEmailAndPassword(auth, String(m.email || ''), String(m.password || '')) + .then(async (cred) => { + await rememberAccount(cred.user); + scheduleFlush(0); + return { user: publicUser(cred.user) }; + }) .catch((err) => ({ error: friendlyAuthError(err) }))); -router.on(Msg.T.PROSCAN_SIGN_OUT, () => - signOut(auth).then(() => ({ ok: true }), (err) => ({ error: err.message }))); +// The same answer whether or not the account exists. +router.on(Msg.T.PROSCAN_RESET_PASSWORD, async (m) => { + const email = String(m.email || '').trim(); + if (!email) return { error: 'Enter your email first.' }; + try { + await sendPasswordResetEmail(auth, email, { url: DASHBOARD_URL }); + } catch (err) { + if (err && err.code === 'auth/invalid-email') return { error: 'That email address does not look valid.' }; + if (err && err.code === 'auth/network-request-failed') return { error: 'Network error. Check your connection.' }; + } + return { ok: true, message: `If an account exists for ${email}, a reset link is on its way.` }; +}); + +router.on(Msg.T.PROSCAN_SIGN_OUT, async () => { + signingOut = true; + try { + await signOut(auth); + await chrome.storage.local.remove(['account', 'authNotice']); + return { ok: true }; + } catch (err) { + return { error: err.message }; + } finally { + signingOut = false; + } +}); router.on(Msg.T.PROSCAN_EXPORT, async () => { if (!Flags.CLOUD_SYNC) return { error: 'Export to ProScan is not available in this version.' }; const user = await currentUser(); if (!user) return { error: 'Sign in to ProScan first.' }; - const { bundle, seqs } = await engine.outboxBundle(); - if (bundle.syncQueue.length === 0) return { ok: true, written: 0, products: 0 }; try { - const result = await syncToCloud(user.uid, bundle); - // Only what was written leaves the outbox; lastValues stays for deltas. - await engine.clearOutbox(seqs); - return { ok: true, ...result }; + const out = await flushNow({ manual: true }); + return { ok: true, ...out, failed: await sync.failed(user.uid).catch(() => out.failed || 0) }; } catch (err) { - console.error('[ProScan] cloud export failed', err); - return { error: (err && err.message) || 'Export failed.' }; + if (isAuthError(err)) return { error: 'Your session expired. Sign in again to keep syncing.', expired: true }; + return { error: 'Could not reach ProScan. Your scans are kept and will sync later.' }; } }); diff --git a/scripts/background/sync-plan.js b/scripts/background/sync-plan.js new file mode 100644 index 0000000..1c9ed77 --- /dev/null +++ b/scripts/background/sync-plan.js @@ -0,0 +1,294 @@ +/** + * @fileoverview What one outbox entry writes to the cloud. Pure: records + * from IndexedDB in, a list of document writes out. + * + * An entry is one page of a run ({kind:'page', pageIndex}) or the end of a + * run ({kind:'run'}). A page writes its chunk, the product and history + * documents of every ASIN on it, the run header and the source. The end of + * a run writes the header and the source again with the final status. + * + * Each write is {path, data, fields} or {path, data, merge: true}. `fields` + * null means replace the whole document; otherwise only those fields are + * written, each replaced whole (Firestore mergeFields). A field given as an + * array is a field path, e.g. ['d', '2026-06-09']. + * + * A product document is a `merge` write, so its sourceIds arrayUnion keeps + * the sources written before (in a mergeFields mask it would be replaced + * whole). `latest`, `prev` and `delta` are still replaced whole: every key + * they can have and this write lacks is sent as a delete, so a price that + * failed to parse is gone from `latest` instead of kept from last week. + * + * The run header and the source are written once per flush for each run, + * on its last entry in that flush (`header`), not once per page. + * + * Firestore rules cap string lengths and list sizes the schema validators + * do not know about. checkRuleCaps() applies the same caps here, so an + * entry the rules would refuse on every try fails planning instead. + * + * @module SyncPlan + */ + +import { + SV, MK, MAX_PAGE_ITEMS, pageIdOf, sourceOf, sourceIdOf, dayKeyOf, expireAtMs, + runStatusOf, pointOf, deltaOf, timeMs, assertValid +} from '../../packages/schema/index.js'; + +const DAY_MS = 86400000; +const same = (v) => v; +/** What a field delete looks like when the caller passes no converter. */ +export const DELETE = Object.freeze({ delete: true }); + +/** Caps from the dashboard's firestore.rules that the schema does not check. */ +export const RULE_CAPS = { + keyword: 300, url: 2000, sellerId: 40, name: 1000, productUrl: 500, img: 1000, + runId: 260, reason: 100, maxPages: 1000, page: 1000, kind: 20, sourceIds: 200 +}; + +const tooLong = (v, max) => typeof v === 'string' && v.length > max; + +/** Throws when `doc` of `kind` breaks a rules cap. */ +export function checkRuleCaps(kind, doc) { + const errs = []; + const c = RULE_CAPS; + if (kind === 'source') { + if (tooLong(doc.keyword, c.keyword)) errs.push(`keyword: at most ${c.keyword} characters`); + if (tooLong(doc.url, c.url)) errs.push(`url: at most ${c.url} characters`); + if (tooLong(doc.sellerId, c.sellerId)) errs.push(`sellerId: at most ${c.sellerId} characters`); + if (tooLong(doc.lastRunId, c.runId)) errs.push(`lastRunId: at most ${c.runId} characters`); + if (!/^[sk]_[A-Za-z0-9_-]{1,200}$/.test(doc.sourceId || '')) errs.push('sourceId: s_ or k_ and 1 to 200 id characters'); + } else if (kind === 'run') { + if (tooLong(doc.reason, c.reason)) errs.push(`reason: at most ${c.reason} characters`); + if (doc.maxPages > c.maxPages) errs.push(`maxPages: at most ${c.maxPages}`); + } else if (kind === 'page') { + if (doc.page > c.page) errs.push(`page: at most ${c.page}`); + if (tooLong(doc.kind, c.kind)) errs.push(`kind: at most ${c.kind} characters`); + } else if (kind === 'product') { + if (tooLong(doc.name, c.name)) errs.push(`name: at most ${c.name} characters`); + if (tooLong(doc.url, c.productUrl)) errs.push(`url: at most ${c.productUrl} characters`); + if (tooLong(doc.img, c.img)) errs.push(`img: at most ${c.img} characters`); + if (doc.latest && tooLong(doc.latest.runId, c.runId)) errs.push(`latest.runId: at most ${c.runId} characters`); + if (tooLong(doc.firstRunId, c.runId)) errs.push(`firstRunId: at most ${c.runId} characters`); + } + if (errs.length) throw new Error(`${kind} document breaks a rules cap: ${errs.join('; ')}`); + return doc; +} + +/** Every key each replaced map of a product document can hold. */ +const MAP_KEYS = { + latest: ['p', 'r', 'v', 'pr', 'rk', 'at', 'runId', 'dayKey'], + prev: ['p', 'r', 'v', 'pr', 'rk', 'at'], + delta: ['p', 'pPct', 'r', 'v', 'days'] +}; + +/** `doc` with the keys its replaced maps lack set to `del()`, for a merge write. */ +function withDeletes(doc, del) { + const out = { ...doc }; + for (const [field, keys] of Object.entries(MAP_KEYS)) { + const v = doc[field]; + if (!v || typeof v !== 'object') continue; + const filled = { ...v }; + for (const k of keys) if (!(k in filled)) filled[k] = del(); + out[field] = filled; + } + return out; +} + +/** Placements of `product` on page `pageIndex`. */ +function onPage(product, pageIndex) { + return (product.placements || []).filter((pl) => pl.page === pageIndex); +} + +const clamp = (v, max) => (typeof v === 'string' && v.length > max ? v.slice(0, max) : v); + +/** The run's source id, day key and source, also for a run from before 2.3. */ +export function runKeys(run) { + const source = run.source || {}; + const found = source.url ? sourceOf(source.url) : { type: source.type || 'keyword', sellerId: source.sellerId || null, keyword: source.keyword || null, url: null }; + const sourceId = run.sourceId || source.sourceId || sourceIdOf(found); + const dayKey = run.dayKey || dayKeyOf(run.startedAt, new Date(run.startedAt).getTimezoneOffset()); + return { + sourceId, + dayKey, + source: { + type: found.type, + sellerId: found.sellerId || null, + keyword: clamp(found.keyword || null, RULE_CAPS.keyword), + url: clamp(found.url || source.url || null, RULE_CAPS.url) + } + }; +} + +/** + * ASINs on this page whose product document may not exist yet: never seen, + * or last seen by nobody signed in to this account. The caller reads them + * and passes back the ones that are missing, which alone get firstSeenAt. + */ +export function firstSeenCandidates(entry, products) { + if (entry.kind !== 'page') return []; + return products + .filter((p) => onPage(p, entry.pageIndex).length > 0) + .filter((p) => !p.prev || p.prev.uid !== entry.uid) + .map((p) => p.asin); +} + +function counters(products) { + const placements = products.flatMap((p) => p.placements || []); + return { + placements: placements.length, + uniqueAsins: products.length, + sponsored: placements.filter((pl) => pl.sponsored).length, + priceParseFailures: products.filter((p) => !Number.isInteger(p.priceCents)).length, + newSeen: products.filter((p) => p.delta && p.delta.isNew).length + }; +} + +function runDoc(run, keys, products, pages, time) { + const ended = !['starting', 'running', 'stopping'].includes(run.state); + const first = pages.find((pg) => pg.pageIndex === 1) || pages[0]; + const total = first && Number.isInteger(first.total) ? first.total : null; + return { + sv: SV, + runId: run.runId, + sourceId: keys.sourceId, + source: keys.source, + mk: MK, + dayKey: keys.dayKey, + startedAt: time(run.startedAt), + finishedAt: ended && run.finishedAt ? time(run.finishedAt) : null, + status: runStatusOf(run.state), + reason: ended ? run.reason || null : null, + pagesDone: pages.length, + maxPages: run.maxPages, + pagesPlanned: ended && run.reason === 'complete' ? pages.length : null, + totalResultsOnSerp: total, + counters: counters(products) + }; +} + +function sourceDoc(run, keys, pages, time) { + const first = pages.find((pg) => pg.pageIndex === 1) || pages[0]; + const doc = { + sv: SV, + sourceId: keys.sourceId, + type: keys.source.type, + sellerId: keys.source.sellerId, + keyword: keys.source.keyword, + url: keys.source.url, + lastRunId: run.runId, + lastScrapedAt: time(run.startedAt) + }; + if (first && Number.isInteger(first.total)) doc.catalogSize = first.total; + return doc; +} + +function pageDoc(run, pageRec, products, time) { + const items = {}; + let truncated = false; + for (const p of products) { + const here = onPage(p, pageRec.pageIndex); + if (here.length === 0) continue; + if (Object.keys(items).length >= MAX_PAGE_ITEMS) { truncated = true; break; } + const organic = here.filter((pl) => !pl.sponsored && Number.isInteger(pl.rank)).map((pl) => pl.rank); + const pt = pointOf({ ...p, organicRank: organic.length ? Math.min(...organic) : null }); + pt.sp = organic.length ? 0 : 1; + items[p.asin] = pt; + } + const doc = { + sv: SV, + runId: run.runId, + page: pageRec.pageIndex, + scrapedAt: time(timeMs(pageRec.scrapedAt) ?? run.startedAt), + expireAt: time(expireAtMs(run.startedAt)), + count: pageRec.count || 0, + placements: pageRec.placements || 0, + kind: pageRec.kind || 'results', + items + }; + if (Number.isInteger(pageRec.total)) doc.total = pageRec.total; + if (truncated) doc.truncated = true; + return doc; +} + +function productDoc(run, keys, p, time, union) { + const at = timeMs(p.scrapedAt) ?? run.startedAt; + const now = pointOf(p); + const prevAt = p.prev ? timeMs(p.prev.scrapedAt) : null; + const prevPt = p.prev ? pointOf({ priceCents: p.prev.priceCents, rating: p.prev.rating, reviewCount: p.prev.reviewCount }) : null; + const days = prevAt !== null ? Math.floor((at - prevAt) / DAY_MS) : null; + const doc = { + sv: SV, + asin: p.asin, + mk: MK, + url: `https://www.amazon.com/dp/${p.asin}`, + latest: { ...now, at: time(at), runId: run.runId, dayKey: keys.dayKey }, + prev: prevPt ? { ...prevPt, ...(prevAt !== null ? { at: time(prevAt) } : {}) } : null, + delta: deltaOf(now, prevPt, days), + sourceIds: union([keys.sourceId]) + }; + // A name or image this card lacked keeps the last one written. + if (typeof p.name === 'string' && p.name) doc.name = p.name; + if (typeof p.img === 'string' && p.img) doc.img = p.img; + return doc; +} + +/** + * The writes for one outbox entry. + * + * @param {Object} args + * @param {{kind:string, runId:string, uid:string, pageIndex?:number}} args.entry + * @param {Object} args.run the run record from IndexedDB + * @param {Object[]} args.products the run's products + * @param {Object[]} args.pages the run's page records + * @param {Set} [args.missing] ASINs with no product document yet + * @param {function(number):*} [args.time] ms to a Firestore Timestamp + * @param {function(string[]):*} [args.union] values to arrayUnion + * @param {function():*} [args.del] a field delete, for merge writes + * @param {boolean} [args.header] also write the run header and the source + * @returns {{path: string[], data: Object, fields?: ?Array, merge?: boolean}[]} + */ +export function planEntry({ entry, run: rec, products, pages, missing = new Set(), time = same, union = same, del = () => DELETE, header: withHeader = true }) { + const run = { ...rec, startedAt: timeMs(rec.startedAt), finishedAt: timeMs(rec.finishedAt) }; + const ws = ['workspaces', entry.uid]; + const keys = runKeys(run); + const writes = []; + + if (entry.kind === 'page') { + const pageRec = pages.find((pg) => pg.pageIndex === entry.pageIndex); + if (pageRec) { + const chunk = pageDoc(run, pageRec, products, time); + assertValid('page', chunk); + checkRuleCaps('page', chunk); + writes.push({ path: [...ws, 'runs', run.runId, 'pages', pageIdOf(pageRec.pageIndex)], data: chunk, fields: null }); + } + for (const p of products) { + if (onPage(p, entry.pageIndex).length === 0) continue; + const doc = productDoc(run, keys, p, time, union); + if (missing.has(p.asin)) { + doc.firstSeenAt = doc.latest.at; + doc.firstRunId = run.runId; + } + assertValid('product', { ...doc, sourceIds: [keys.sourceId] }); + checkRuleCaps('product', doc); + writes.push({ path: [...ws, 'products', p.asin], data: withDeletes(doc, del), merge: true }); + + const point = pointOf(p); + assertValid('history', { sv: SV, asin: p.asin, d: { [keys.dayKey]: point } }); + writes.push({ + path: [...ws, 'products', p.asin, 'history', 'daily'], + data: { sv: SV, asin: p.asin, d: { [keys.dayKey]: point } }, + fields: ['sv', 'asin', ['d', keys.dayKey]] + }); + } + } + + if (!withHeader && entry.kind !== 'run') return writes; + const head = runDoc(run, keys, products, pages, time); + assertValid('run', head); + checkRuleCaps('run', head); + writes.push({ path: [...ws, 'runs', run.runId], data: head, fields: Object.keys(head) }); + const source = sourceDoc(run, keys, pages, time); + assertValid('source', source); + checkRuleCaps('source', source); + writes.push({ path: [...ws, 'sources', keys.sourceId], data: source, fields: Object.keys(source) }); + return writes; +} diff --git a/scripts/background/sync.js b/scripts/background/sync.js index 4dedd8d..4411a1f 100644 --- a/scripts/background/sync.js +++ b/scripts/background/sync.js @@ -1,187 +1,233 @@ /** - * @fileoverview Sync consumer — drains the durable syncQueue into the - * signed-in user's Firestore workspace (workspaces/{uid}), converting the - * producer's shapes to the cloud schema (proscan-web docs/architecture/ - * data-model.md §4) so every write passes firestore.rules. + * @fileoverview Cloud sync: drains the IndexedDB outbox into the signed-in + * user's workspace, one entry at a time. * - * The producer (scraper.js/storage.js/delta.js) stamps each product with - * priceCents + a delta block + runId/pageIndex and pushes it to - * chrome.storage.local 'syncQueue'. It does NOT carry several fields the cloud - * schema requires — this module derives them at write time: - * - canonical sourceId (s_{sellerId} | k_{slug}) from scrapeRunMeta - * - canonical runId ({sourceId}_{startEpochMs}) - * - dayKey (UTC date of run start; rules REQUIRE it as string) - * - mk ('US' — only marketplace today) - * - delta.pPct (percent change; producer only has absolute cents) - * - ISO strings -> Firestore Timestamp - * - isPrime boolean -> pr 0/1 - * Money stays integer cents. Unknown numerics are OMITTED (never written as - * null) so they never violate the rules' `is int` checks. Dashboard-owned - * fields (lead/verdict/tags/spread) are never touched — set(merge) preserves - * them. + * Each entry (one page of a run, or the end of a run) is planned by + * sync-plan.js, committed, and only then deleted. Entries queued while a + * flush runs are picked up before it returns. An entry belongs to the + * account that was signed in when it was queued and is never written into + * another account's workspace. + * + * Every write is idempotent, so an entry that fails halfway is simply + * written again on the next flush. + * + * The rules can refuse one entry for good (permission-denied or + * invalid-argument on a document the schema allowed). That entry is marked + * failed and skipped, so the entries behind it still sync; a flush with + * `retryFailed` (Export to ProScan) tries the failed ones again. When every + * entry of a flush is refused, the cause is the account or the rules, not + * one entry, so nothing is marked and the error goes to the caller. + * + * Authored as ESM and bundled into the service worker by esbuild. The + * contract test runs this same file in Node against the emulators. * - * Bundled into the service worker by esbuild. * @module Sync */ import { - writeBatch, doc, - serverTimestamp, + getDoc, + writeBatch, Timestamp, arrayUnion, + deleteField, + FieldPath, } from 'firebase/firestore'; -import { db } from './firebase-init.js'; - -const slugify = (s) => - String(s).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); -const tsOf = (iso) => Timestamp.fromDate(new Date(iso)); -const round1 = (n) => Math.round(n * 10) / 10; -const intOrUndef = (n) => - typeof n === 'number' && Number.isFinite(n) ? Math.round(n) : undefined; - -function deriveSourceId(meta) { - if (meta && meta.sellerId) return `s_${meta.sellerId}`; - if (meta && meta.keyword) return `k_${slugify(meta.keyword)}`; - return 'k_unknown'; -} +import { planEntry, firstSeenCandidates } from './sync-plan.js'; + +const FIRESTORE = { doc, getDoc, writeBatch, Timestamp, arrayUnion, deleteField, FieldPath }; + +/** Firestore allows 500 writes per batch; stay under it. */ +export const BATCH_LIMIT = 450; -/** Compact observation point {p,r,v,pr} — schema's history/latest shape. - * Omits unknown numerics so the rules' `is int` checks never see a null. */ -function pointFrom(q) { - const pt = { - p: intOrUndef(q.priceCents), - r: typeof q.rating === 'number' && q.rating > 0 ? round1(q.rating) : undefined, - v: typeof q.reviewCount === 'number' && q.reviewCount > 0 ? Math.round(q.reviewCount) : undefined, - pr: q.isPrime ? 1 : 0, - }; - Object.keys(pt).forEach((k) => pt[k] === undefined && delete pt[k]); - return pt; +/** + * Errors that mean the account's session is gone, not the network. A + * permission-denied is the rules refusing a document, not a lost session. + */ +const AUTH_CODES = ['unauthenticated', 'auth/user-token-expired', 'auth/user-disabled', 'auth/invalid-user-token']; + +export function isAuthError(err) { + return !!err && AUTH_CODES.includes(err.code); } -/** Convert the producer delta {dPriceCents,dRating,dReviews} → schema - * {p,pPct,r,v}. Returns undefined for first-sight / empty deltas. */ -function deltaBlock(q) { - const d = q.delta; - if (!d || d.isNew) return undefined; - const out = {}; - if (typeof d.dPriceCents === 'number') { - out.p = Math.round(d.dPriceCents); - const prev = typeof q.priceCents === 'number' ? q.priceCents - d.dPriceCents : null; - if (prev && prev !== 0) out.pPct = round1((d.dPriceCents / prev) * 100); - } - if (typeof d.dRating === 'number') out.r = round1(d.dRating); - if (typeof d.dReviews === 'number') out.v = Math.round(d.dReviews); - return Object.keys(out).length ? out : undefined; +/** The rules or the backend refused this write for good; a retry would fail the same way. */ +const REFUSAL_CODES = ['permission-denied', 'invalid-argument']; + +export function isRefusal(err) { + return !!err && REFUSAL_CODES.includes(err.code); } +const EMPTY = () => ({ entries: 0, pages: 0, runs: 0, products: 0, writes: 0, failed: 0 }); + /** - * Drain the queue into workspaces/{uid}. Idempotent: every write is - * set(merge) of absolute values + one arrayUnion, so a retried export - * rewrites byte-identical docs. - * - * @param {string} uid signed-in user's uid (== workspace id) - * @param {{syncQueue?: object[], scrapeRunMeta?: object, scrapeRunPages?: object[]}} bundle - * @returns {Promise<{written: number, runId: string|null, products: number}>} + * @param {Object} deps + * @param {Object} deps.db - the Firestore instance + * @param {function(): Promise} deps.openStore - resolves to the db.js api + * @param {Object} [deps.fs] - firebase/firestore functions, for tests + * @param {function(Object): Promise} [deps.onBatch] - called before each commit, for tests */ -export async function syncToCloud(uid, bundle) { - const queue = (bundle && bundle.syncQueue) || []; - if (!queue.length) return { written: 0, runId: null, products: 0 }; - - const meta = (bundle && bundle.scrapeRunMeta) || {}; - const sourceId = deriveSourceId(meta); - const startMs = meta.startedAt ? Date.parse(meta.startedAt) : Date.now(); - const runId = `${sourceId}_${startMs}`; - const dayKey = new Date(startMs).toISOString().slice(0, 10); - const mk = 'US'; - - // ── products + history, chunked under the 500-writes/batch limit ── - const CHUNK = 200; // 2 writes per product → 400 writes/batch - let products = 0; - for (let i = 0; i < queue.length; i += CHUNK) { - const batch = writeBatch(db); - for (const q of queue.slice(i, i + CHUNK)) { - if (!q || !q.asin) continue; - const at = q.scrapedAt ? tsOf(q.scrapedAt) : serverTimestamp(); - - const payload = { - asin: q.asin, // rules: doc id must equal this - mk, - url: `https://www.amazon.com/dp/${q.asin}`, - latest: { ...pointFrom(q), at, runId, dayKey }, // rules require latest.dayKey:string - sourceIds: arrayUnion(sourceId), - }; - if (typeof q.name === 'string' && q.name) payload.name = q.name; - const d = deltaBlock(q); - if (d) payload.delta = d; - if (q.delta && q.delta.isNew) { - // immutable first-sight stamps — only on first sight, else merge would clobber - payload.firstSeenAt = at; - payload.firstRunId = runId; +export function createSync({ db, openStore, fs = FIRESTORE, batchLimit = BATCH_LIMIT, onBatch = null, log = console }) { + let running = null; + let again = false; + + const ref = (path) => fs.doc(db, ...path); + const time = (ms) => fs.Timestamp.fromMillis(ms); + // Product writes need field deletes; without them every page would fail planning and be dropped. + if (typeof fs.deleteField !== 'function') throw new Error('createSync: fs.deleteField is missing'); + const union = (vals) => fs.arrayUnion(...vals); + const del = () => fs.deleteField(); + const field = (f) => (Array.isArray(f) ? new fs.FieldPath(...f) : f); + + /** Entries queued for `uid`, oldest first; failed ones only when `failed` is true. */ + async function mine(store, uid, { failed = false } = {}) { + const all = await store.getAll('outbox'); + return all + .filter((e) => e.uid === uid && (e.kind === 'page' || e.kind === 'run') && !!e.failed === failed) + .sort((a, b) => a.seq - b.seq); + } + + /** Product documents among `asins` that do not exist yet. */ + async function missingOf(uid, asins) { + const missing = new Set(); + for (let i = 0; i < asins.length; i += 25) { + const part = asins.slice(i, i + 25); + const snaps = await Promise.all(part.map((a) => fs.getDoc(ref(['workspaces', uid, 'products', a])))); + snaps.forEach((snap, j) => { if (!snap.exists()) missing.add(part[j]); }); + } + return missing; + } + + /** + * Writes one entry, with the run header and source when `header` is true. + * Returns how many writes it took, or null if it had nothing to write. + */ + async function commitEntry(store, entry, header) { + const run = await store.get('runs', entry.runId); + if (!run) return null; + const [products, pages] = await Promise.all([store.runProducts(entry.runId), store.runPages(entry.runId)]); + const missing = await missingOf(entry.uid, firstSeenCandidates(entry, products)); + let writes; + try { + writes = planEntry({ entry, run, products, pages, missing, time, union, del, header }); + } catch (err) { + // Planning is pure, so a retry would fail the same way and hold up the queue. + log.warn('[ProScan] Dropped an outbox entry that cannot be written:', err.message); + return null; + } + + for (let i = 0; i < writes.length; i += batchLimit) { + const batch = fs.writeBatch(db); + for (const w of writes.slice(i, i + batchLimit)) { + if (w.merge) batch.set(ref(w.path), w.data, { merge: true }); + else if (w.fields) batch.set(ref(w.path), w.data, { mergeFields: w.fields.map(field) }); + else batch.set(ref(w.path), w.data); + } + if (onBatch) await onBatch({ entry, writes: Math.min(batchLimit, writes.length - i) }); + await batch.commit(); + } + return { writes: writes.length, products: products.filter((p) => (p.placements || []).some((pl) => pl.page === entry.pageIndex)).length }; + } + + async function drain(uid, state) { + const store = await openStore(); + const totals = EMPTY(); + const runs = new Set(); + // New entries can arrive while we write; keep going until none are left. + for (let round = 0; round < 1000; round++) { + const entries = await mine(store, uid); + if (entries.length === 0) break; + // The header and source go with each run's last entry in this round. + const lastOf = new Map(entries.map((e, i) => [e.runId, i])); + const refused = []; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + let done; + try { + done = await commitEntry(store, entry, lastOf.get(entry.runId) === i); + } catch (err) { + if (!isRefusal(err)) throw err; + refused.push({ entry, err }); + continue; + } + state.through = true; + await store.write([{ store: 'outbox', delete: entry.seq }]); + totals.entries++; + if (!done) continue; + runs.add(entry.runId); + totals.writes += done.writes; + if (entry.kind === 'page') { + totals.pages++; + totals.products += done.products; + } + } + if (refused.length === 0) continue; + // Entries refused before stay failed. For the others, nothing getting + // through at all points at the account or the rules, not these entries. + const before = refused.filter(({ entry }) => state.retried.has(entry.seq)); + const fresh = refused.filter(({ entry }) => !state.retried.has(entry.seq)); + const mark = state.through ? refused : before; + if (mark.length) { + await store.write(mark.map(({ entry, err }) => ({ + store: 'outbox', put: { ...entry, failed: { code: err.code, at: Date.now() } }, + }))); } - batch.set(doc(db, 'workspaces', uid, 'products', q.asin), payload, { merge: true }); - - // date-keyed history point (deep-merges into the d-map) - batch.set( - doc(db, 'workspaces', uid, 'products', q.asin, 'history', 'daily'), - { asin: q.asin, d: { [dayKey]: pointFrom(q) } }, - { merge: true }, - ); - products++; + if (!state.through && fresh.length) throw fresh[0].err; + totals.failed += mark.length; + log.warn(`[ProScan] The rules refused ${refused.length} outbox entr${refused.length === 1 ? 'y' : 'ies'}; the rest go on.`); } - await batch.commit(); + totals.runs = runs.size; + return totals; + } + + /** Puts `uid`'s failed entries back in line. */ + async function unfail(uid) { + const store = await openStore(); + const failed = await mine(store, uid, { failed: true }); + if (failed.length) await store.write(failed.map(({ failed: _f, ...entry }) => ({ store: 'outbox', put: entry }))); + return new Set(failed.map((e) => e.seq)); + } + + /** + * Writes everything queued for `uid`. One flush at a time; a call during + * a flush makes it look again once more before it resolves. + * + * @param {string} uid + * @param {{retryFailed?: boolean}} [opts] - also try entries the rules refused before + * @returns {Promise<{entries:number, pages:number, runs:number, products:number, writes:number, failed:number}>} + */ + function flush(uid, { retryFailed = false } = {}) { + if (!uid) return Promise.resolve(EMPTY()); + if (running) { + again = true; + return running; + } + running = (async () => { + const totals = EMPTY(); + const state = { through: false, retried: retryFailed ? await unfail(uid) : new Set() }; + do { + again = false; + const t = await drain(uid, state); + for (const k of Object.keys(totals)) totals[k] += t[k]; + } while (again); + return totals; + })().catch((err) => { + log.warn('[ProScan] Sync stopped:', err && (err.code || err.message)); + throw err; + }).finally(() => { running = null; }); + return running; + } + + /** How many entries are waiting for `uid`, not counting failed ones. */ + async function pending(uid) { + if (!uid) return 0; + return (await mine(await openStore(), uid)).length; + } + + /** How many of `uid`'s entries the rules refused. */ + async function failedCount(uid) { + if (!uid) return 0; + return (await mine(await openStore(), uid, { failed: true })).length; } - // ── run header + source spine (once per drain) ── - const head = writeBatch(db); - head.set( - doc(db, 'workspaces', uid, 'runs', runId), - { - runId, - sourceId, // rules require string - source: { - type: meta.type || 'keyword', - sellerId: meta.sellerId ?? null, - keyword: meta.keyword ?? null, - url: meta.url ?? null, - }, - mk, - dayKey, // rules require string - startedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - finishedAt: serverTimestamp(), - status: 'complete', // rules enum - pagesDone: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - pagesPlanned: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - counters: { - // Each queued product is one ASIN; its placements list every card it had - placements: queue.reduce((n, q) => n + (Array.isArray(q.placements) ? q.placements.length : 1), 0), - uniqueAsins: new Set(queue.map((q) => q.asin)).size, - sponsored: queue.reduce( - (n, q) => n + (Array.isArray(q.placements) ? q.placements.filter((pl) => pl.sponsored).length : q.sponsored ? 1 : 0), - 0, - ), - priceParseFailures: queue.filter((q) => q.priceCents == null).length, - newSeen: queue.filter((q) => q.delta && q.delta.isNew).length, - }, - }, - { merge: true }, - ); - head.set( - doc(db, 'workspaces', uid, 'sources', sourceId), - { - sourceId, - type: meta.type === 'storefront' ? 'storefront' : 'keyword', // rules enum - sellerId: meta.sellerId ?? null, - keyword: meta.keyword ?? null, - url: meta.url ?? null, - lastRunId: runId, - lastScrapedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - // cadenceDays intentionally omitted — rules default it; never stomp a - // dashboard-set cadence on re-scan. - }, - { merge: true }, - ); - await head.commit(); - - return { written: products * 2 + 2, runId, products }; + return { flush, pending, failed: failedCount }; } diff --git a/scripts/lib/flags.js b/scripts/lib/flags.js index 1a5a98b..883bcef 100644 --- a/scripts/lib/flags.js +++ b/scripts/lib/flags.js @@ -1,17 +1,16 @@ /** * @fileoverview Build flags. * - * CLOUD_SYNC is off in 2.1 and 2.2. Nothing goes into the sync outbox, the - * service worker refuses PROSCAN_EXPORT, and the popup hides the ProScan - * sign-in and the Export to ProScan button. Sync comes back with the - * per-run outbox (audit report, phase 4). + * CLOUD_SYNC turns on the ProScan account and sync: signed-in scans go + * into the outbox and flush to the dashboard, and the popup shows sign-in + * and Export to ProScan. It was off in 2.1 and 2.2 and is on from 2.3. * * Loaded as a plain global (`Flags`) in the popup, and as a CommonJS * module in Jest and the bundled service worker. */ const Flags = { - CLOUD_SYNC: false + CLOUD_SYNC: true }; if (typeof module !== 'undefined' && module.exports) { diff --git a/scripts/lib/messages.js b/scripts/lib/messages.js index 90d08be..eade46b 100644 --- a/scripts/lib/messages.js +++ b/scripts/lib/messages.js @@ -37,6 +37,7 @@ const Msg = (() => { PROSCAN_AUTH_STATE: { to: 'worker', from: 'page' }, PROSCAN_SIGN_IN: { to: 'worker', from: 'page' }, PROSCAN_SIGN_OUT: { to: 'worker', from: 'page' }, + PROSCAN_RESET_PASSWORD: { to: 'worker', from: 'page' }, PROSCAN_EXPORT: { to: 'worker', from: 'page' }, // Worker or popup -> content script diff --git a/scripts/lib/parsers.js b/scripts/lib/parsers.js index 5a524c2..d242d4b 100644 --- a/scripts/lib/parsers.js +++ b/scripts/lib/parsers.js @@ -47,6 +47,7 @@ const Parsers = (() => { productLink: '.a-link-normal.s-no-outline', sponsored: '.puis-sponsored-label-text, .s-sponsored-label-text, [data-component-type="sp-sponsored-result"], a[href*="/sspa/"]', primeBadge: '.a-icon-prime, .s-prime', + image: 'img.s-image', nextPage: '.s-pagination-next:not(.s-pagination-disabled)', nextPageLink: 'a.s-pagination-next[href]:not(.s-pagination-disabled)', nextPageDisabled: '.s-pagination-next.s-pagination-disabled', @@ -188,6 +189,13 @@ const Parsers = (() => { element.querySelector(SEARCH_SELECTORS.sponsored) !== null; } + /** The card's product image on Amazon's image host, or null. */ + function extractImage(element) { + const img = element.querySelector(SEARCH_SELECTORS.image); + const src = img ? img.getAttribute('src') : null; + return src && /^https:\/\/(m\.media-amazon\.com|images-na\.ssl-images-amazon\.com)\//.test(src) ? src : null; + } + /** The product page for an ASIN. Card links can be expiring sspa ad redirects. */ function productUrl(asin) { return `https://www.amazon.com/dp/${asin}`; @@ -219,6 +227,7 @@ const Parsers = (() => { isPrime: isPrime, sponsored: isSponsored(listing), url: productUrl(asin), + img: extractImage(listing), scrapedAt: new Date().toISOString() }; } @@ -476,6 +485,7 @@ const Parsers = (() => { hasPrimeBadge, isSponsored, productUrl, + extractImage, scrapeProduct, getTotalResults, getNextPageUrl, diff --git a/tests/contract/sync.contract.mjs b/tests/contract/sync.contract.mjs new file mode 100644 index 0000000..2a8f533 --- /dev/null +++ b/tests/contract/sync.contract.mjs @@ -0,0 +1,325 @@ +// The sync contract: the real engine fills IndexedDB (fake-indexeddb), the +// real sync module drains it into the Firestore emulator, and the dashboard +// repo's firestore.rules judge every write. Run through tools/contract.mjs, +// which starts the emulators. +// +// Queue sizes 1, 201 and 600, a page appended in the middle of a flush, +// replace semantics across runs, create-only firstSeenAt, a write count +// per run, a product in two sources, and an entry the rules refuse. + +import 'fake-indexeddb/auto'; +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { initializeApp, deleteApp } from 'firebase/app'; +import { + getAuth, connectAuthEmulator, createUserWithEmailAndPassword, signInWithEmailAndPassword, signOut, +} from 'firebase/auth'; +import { + initializeFirestore, connectFirestoreEmulator, doc, getDoc, collection, getCountFromServer, terminate, +} from 'firebase/firestore'; +import { createSync, isAuthError } from '../../scripts/background/sync.js'; +import * as Schema from '../../packages/schema/index.js'; + +const require = createRequire(import.meta.url); +const DB = require('../../scripts/background/db.js'); +const { createEngine } = require('../../scripts/background/engine.js'); + +const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST; +const FS_HOST = process.env.FIRESTORE_EMULATOR_HOST; +if (!AUTH_HOST || !FS_HOST) throw new Error('run through tools/contract.mjs, which starts the emulators'); + +const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }); +const auth = getAuth(app); +const db = initializeFirestore(app, {}); +connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); +connectFirestoreEmulator(db, FS_HOST.split(':')[0], Number(FS_HOST.split(':')[1])); + +const quiet = { log() {}, warn() {}, error() {} }; +const DAY = 86400000; +let dbSeq = 0; + +function memoryArea(init = {}) { + let data = JSON.parse(JSON.stringify(init)); + return { + async get(keys) { + const list = keys == null ? Object.keys(data) : [].concat(keys); + return Object.fromEntries(list.filter((k) => data[k] !== undefined).map((k) => [k, data[k]])); + }, + async set(items) { Object.assign(data, JSON.parse(JSON.stringify(items))); }, + async remove(keys) { [].concat(keys).forEach((k) => delete data[k]); }, + }; +} + +/** The real engine over a fresh IndexedDB, with a tab that always answers. */ +function extension(uid) { + const name = `contract-${process.pid}-${++dbSeq}`; + const clock = { t: 0 }; + let url = ''; + const chrome = { + storage: { session: memoryArea(), local: memoryArea({ account: { uid }, settings: { maxPages: 400 } }) }, + runtime: { lastError: null }, + tabs: { + sendMessage(tabId, message, opts, cb) { + setImmediate(() => cb(message.type === 'PING' ? { ok: true, kind: 'results', url } : { ok: true })); + }, + async update() { return {}; }, + async get(id) { return { id, url }; }, + }, + }; + const engine = createEngine({ + chrome, openDb: () => DB.open({ name }), now: () => clock.t, random: () => 0, + setTimer: () => 0, clearTimer: () => {}, flags: { CLOUD_SYNC: true }, log: quiet, + }); + const sender = { tab: { id: 1 } }; + + /** Starts a keyword run at `startMs`, or a run on `at` when given. */ + async function start(keyword, startMs, at = null) { + clock.t = startMs; + url = at || `https://www.amazon.com/s?k=${encodeURIComponent(keyword)}`; + const resp = await engine.start({ tabId: 1 }); + assert.equal(resp.ok, true, JSON.stringify(resp)); + return resp.runId; + } + + /** Reports page `page` of the live run; `last` ends the run. */ + async function page(runId, pageNo, products, { last = false, total = 0 } = {}) { + if (pageNo > 1) { + clock.t += 5000; + await engine.tick(); + } + const result = { + kind: last ? 'last' : 'results', + products: products.map((p) => ({ ...p, scrapedAt: new Date(clock.t).toISOString() })), + placements: products.length, + nextHref: last ? null : `${url}&page=${pageNo + 1}`, + total, + fill: { asin: 1, title: 1, price: 1 }, + }; + const resp = await engine.pageResult({ runId, page: pageNo, url: pageNo === 1 ? url : `${url}&page=${pageNo}`, result }, sender); + assert.equal(resp.ok, true, JSON.stringify(resp)); + } + + /** A whole run of `products`, `perPage` to a page. */ + async function scrape(keyword, startMs, products, perPage, at = null) { + const runId = await start(keyword, startMs, at); + const pages = Math.ceil(products.length / perPage); + for (let i = 0; i < pages; i++) { + await page(runId, i + 1, products.slice(i * perPage, (i + 1) * perPage), { last: i === pages - 1, total: products.length }); + } + return runId; + } + + const sync = (opts = {}) => createSync({ db, openStore: () => engine.db(), log: quiet, ...opts }); + return { engine, start, page, scrape, sync, store: () => engine.db() }; +} + +const asinOf = (i) => `B0${i.toString(36).toUpperCase().padStart(8, '0')}`; +function products(n, { price = (i) => 1000 + i } = {}) { + return Array.from({ length: n }, (_, i) => { + const cents = price(i); + return { + asin: asinOf(i), name: `Product ${i}`, price: cents === null ? null : `$${(cents / 100).toFixed(2)}`, + priceCents: cents, currency: cents === null ? null : 'USD', rating: 4.5, reviewCount: 100 + i, isPrime: i % 2 === 0, + sponsored: false, url: `https://www.amazon.com/dp/${asinOf(i)}`, + img: `https://m.media-amazon.com/images/I/${i}.jpg`, + placements: [{ position: 1, sponsored: false, rank: 1 }], + }; + }); +} + +/** + * Writes a run takes: per page the chunk and 2 per product (document and + * history); the header and source once per flush round the run had entries in. + */ +const expectedWrites = (pageSizes, rounds = 1) => pageSizes.reduce((n, k) => n + 1 + 2 * k, 0) + 2 * rounds; + +async function account(email) { + const cred = await createUserWithEmailAndPassword(auth, email, 'contract-pass-1'); + return cred.user.uid; +} + +const ws = (uid, ...rest) => doc(db, 'workspaces', uid, ...rest); +const count = async (uid, ...path) => (await getCountFromServer(collection(db, 'workspaces', uid, ...path))).data().count; + +before(async () => { + const rules = process.env.PROSCAN_CONTRACT_RULES || '(unknown)'; + console.log(`# rules: ${rules}`); +}); + +after(async () => { + await signOut(auth).catch(() => {}); + await terminate(db).catch(() => {}); + await deleteApp(app).catch(() => {}); +}); + +test('queue size 1: one product on one page', async () => { + const uid = await account(`one-${Date.now()}@contract.test`); + const ext = extension(uid); + const start = Date.parse('2026-06-09T14:00:00Z'); + const runId = await ext.scrape('single mug', start, products(1), 60); + assert.equal(runId, `k_single-mug_${start}`); + + const totals = await ext.sync().flush(uid); + assert.deepEqual(totals, { entries: 2, pages: 1, runs: 1, products: 1, writes: expectedWrites([1]), failed: 0 }); + assert.equal(await (await ext.store()).count('outbox'), 0); + + const p = (await getDoc(ws(uid, 'products', asinOf(0)))).data(); + assert.deepEqual(Schema.validateProduct({ ...p }), []); + assert.equal(p.latest.p, 1000); + assert.equal(p.latest.runId, runId); + assert.equal(p.firstRunId, runId); + assert.equal(p.img, 'https://m.media-amazon.com/images/I/0.jpg'); + const run = (await getDoc(ws(uid, 'runs', runId))).data(); + assert.deepEqual(Schema.validateRun(run), []); + assert.equal(run.status, 'complete'); + assert.equal(run.pagesPlanned, 1); + const chunk = (await getDoc(ws(uid, 'runs', runId, 'pages', 'p0001'))).data(); + assert.deepEqual(Schema.validatePage(chunk), []); + assert.deepEqual(Object.keys(chunk.items), [asinOf(0)]); + const hist = (await getDoc(ws(uid, 'products', asinOf(0), 'history', 'daily'))).data(); + assert.deepEqual(Schema.validateHistory(hist), []); + const src = (await getDoc(ws(uid, 'sources', 'k_single-mug'))).data(); + assert.deepEqual(Schema.validateSource(src), []); + assert.equal(src.lastRunId, runId); +}); + +test('queue size 201, with a page appended in the middle of the flush', async () => { + const uid = await account(`mid-${Date.now()}@contract.test`); + const ext = extension(uid); + const all = products(201); + const start = Date.parse('2026-06-10T09:00:00Z'); + const runId = await ext.start('yoga mat', start); + for (let i = 0; i < 4; i++) await ext.page(runId, i + 1, all.slice(i * 48, (i + 1) * 48), { total: 201 }); + + let appended = false; + const sync = ext.sync({ + onBatch: async () => { + if (appended) return; + appended = true; + // The last page lands while page 1 is being written. + await ext.page(runId, 5, all.slice(192), { last: true, total: 201 }); + }, + }); + const totals = await sync.flush(uid); + assert.equal(appended, true); + assert.equal(totals.pages, 5); + assert.equal(totals.products, 201); + assert.equal(totals.writes, expectedWrites([48, 48, 48, 48, 9], 2)); + assert.equal(await (await ext.store()).count('outbox'), 0); + + assert.equal(await count(uid, 'products'), 201); + assert.equal(await count(uid, 'runs', runId, 'pages'), 5); + const run = (await getDoc(ws(uid, 'runs', runId))).data(); + assert.equal(run.status, 'complete'); + assert.equal(run.pagesDone, 5); + assert.equal(run.counters.uniqueAsins, 201); + assert.equal(run.totalResultsOnSerp, 201); + const last = (await getDoc(ws(uid, 'products', asinOf(200)))).data(); + assert.equal(last.latest.p, 1200); +}); + +test('queue size 600: two runs a week apart replace latest and delta, and keep firstSeenAt', async () => { + const uid = await account(`six-${Date.now()}@contract.test`); + const ext = extension(uid); + const day1 = Date.parse('2026-06-01T15:00:00Z'); + const day8 = day1 + 7 * DAY; + const run1 = await ext.scrape('garden hose', day1, products(300), 60); + // Week two: every 10th price fails to parse, the rest drop by 100 cents. + const run2 = await ext.scrape('garden hose', day8, products(300, { price: (i) => (i % 10 === 0 ? null : 900 + i) }), 60); + + const totals = await ext.sync().flush(uid); + assert.equal(totals.pages, 10); + assert.equal(totals.runs, 2); + assert.equal(totals.products, 600); + assert.equal(totals.writes, 2 * expectedWrites([60, 60, 60, 60, 60])); + assert.equal(await count(uid, 'products'), 300); + + const failed = (await getDoc(ws(uid, 'products', asinOf(10)))).data(); + assert.equal(failed.latest.runId, run2); + assert.equal('p' in failed.latest, false, 'a price that failed to parse is not last week\'s price (F-21)'); + assert.equal('p' in (failed.delta || {}), false, 'no price delta without a price (F-21)'); + assert.equal(failed.firstRunId, run1); + + const moved = (await getDoc(ws(uid, 'products', asinOf(11)))).data(); + assert.equal(moved.latest.p, 911); + assert.equal(moved.prev.p, 1011); + assert.deepEqual(moved.delta, { p: -100, pPct: -9.9, r: 0, v: 0, days: 7 }); + assert.equal(moved.firstRunId, run1); + assert.equal(moved.firstSeenAt.toMillis(), day1); + + const hist = (await getDoc(ws(uid, 'products', asinOf(11), 'history', 'daily'))).data(); + assert.deepEqual(Object.keys(hist.d).sort(), [Schema.dayKeyOf(day1, new Date(day1).getTimezoneOffset()), Schema.dayKeyOf(day8, new Date(day8).getTimezoneOffset())].sort()); + + for (const runId of [run1, run2]) { + const run = (await getDoc(ws(uid, 'runs', runId))).data(); + assert.equal(run.status, 'complete'); + assert.equal(run.sourceId, 'k_garden-hose'); + assert.equal(await count(uid, 'runs', runId, 'pages'), 5); + } + assert.equal((await getDoc(ws(uid, 'runs', run1))).data().counters.newSeen, 300); + assert.equal((await getDoc(ws(uid, 'runs', run2))).data().counters.newSeen, 0); + assert.equal((await getDoc(ws(uid, 'sources', 'k_garden-hose'))).data().lastRunId, run2); +}); + +test('a reinstall does not move firstSeenAt (F-29b)', async () => { + const uid = await account(`re-${Date.now()}@contract.test`); + const first = extension(uid); + const day1 = Date.parse('2026-05-01T12:00:00Z'); + const run1 = await first.scrape('desk lamp', day1, products(5), 60); + await first.sync().flush(uid); + + // A fresh install on another computer: no lastValues, every product looks new. + const second = extension(uid); + await second.scrape('desk lamp', day1 + 30 * DAY, products(5), 60); + await second.sync().flush(uid); + + const p = (await getDoc(ws(uid, 'products', asinOf(3)))).data(); + assert.equal(p.firstRunId, run1); + assert.equal(p.firstSeenAt.toMillis(), day1); +}); + +test('another account cannot write this account\'s queue, and the entries stay', async () => { + const owner = await account(`a-${Date.now()}@contract.test`); + const ext = extension(owner); + await ext.scrape('phone case', Date.parse('2026-06-12T10:00:00Z'), products(3), 60); + await signOut(auth); + await account(`b-${Date.now()}@contract.test`); + + await assert.rejects(ext.sync().flush(owner), (err) => err.code === 'permission-denied' && !isAuthError(err)); + assert.equal(await (await ext.store()).count('outbox'), 2); +}); + +test('a product seen by a storefront and a keyword run lists both sources (NEW-SYNC-1)', async () => { + const uid = await account(`two-${Date.now()}@contract.test`); + const ext = extension(uid); + const day = Date.parse('2026-06-14T10:00:00Z'); + await ext.scrape('store', day, products(6), 60, 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2'); + await ext.sync().flush(uid); + // The keyword run sees the first 3 again. + await ext.scrape('water bottle', day + 60000, products(3), 60); + await ext.sync().flush(uid); + + const both = (await getDoc(ws(uid, 'products', asinOf(1)))).data(); + assert.deepEqual([...both.sourceIds].sort(), ['k_water-bottle', 's_A3K9XELT4QZ6M2']); + const one = (await getDoc(ws(uid, 'products', asinOf(5)))).data(); + assert.deepEqual(one.sourceIds, ['s_A3K9XELT4QZ6M2']); +}); + +test('an entry the rules refuse is set aside and the next scan still syncs (EXT9-1)', async () => { + const uid = await account(`skew-${Date.now()}@contract.test`); + const ext = extension(uid); + // A clock two hours fast: the rules refuse times over an hour ahead. + const ahead = Date.now() + 2 * 60 * 60 * 1000; + const bad = await ext.scrape('fast clock', ahead, products(2), 60); + // Other ASINs: these would carry the fast clock in prev.at. + const good = await ext.scrape('garden hose', Date.now() - 60000, products(4).slice(2), 60); + + const totals = await ext.sync().flush(uid); + assert.equal(totals.failed, 2); + assert.equal((await getDoc(ws(uid, 'runs', good))).exists(), true); + assert.equal((await getDoc(ws(uid, 'runs', bad))).exists(), false); + const sync = ext.sync(); + assert.equal(await sync.pending(uid), 0); + assert.equal(await sync.failed(uid), 2); +}); diff --git a/tests/e2e/scrape.spec.mjs b/tests/e2e/scrape.spec.mjs index 833a124..349c325 100644 --- a/tests/e2e/scrape.spec.mjs +++ b/tests/e2e/scrape.spec.mjs @@ -255,8 +255,7 @@ async function twoRuns(ext, store, done = () => true) { (x.results || []).some((r) => r.name.startsWith('yoga mat')) && done(x), { timeout: 30000 }); } -test('with cloud sync off, two runs queue nothing and keep their deltas (F-26)', async ({ ext }) => { - test.skip(Flags.CLOUD_SYNC, 'cloud sync is on in this build'); +test('signed out, two runs queue nothing and keep their deltas (F-26)', async ({ ext }) => { await serveAmazon(ext.context, simplePlan(2)); const store = await extPage(ext); const s = await twoRuns(ext, store); @@ -278,14 +277,37 @@ test('with cloud sync off, the popup shows no sign-in and no Export to ProScan', expect(resp.error).toMatch(/not available/); }); -test('two runs without an export keep their own attribution', async ({ ext }) => { - test.skip(!Flags.CLOUD_SYNC, 'F-20: cloud sync is off until 2.3, so nothing is queued to attribute'); +test('with cloud sync on, sign-in is folded away and offers sign-up and a password reset', async ({ ext }) => { + test.skip(!Flags.CLOUD_SYNC, 'cloud sync is off in this build'); + const popup = await extPage(ext); + await expect(popup.locator('#actionButton')).toBeVisible(); + await expect(popup.locator('#authPanel')).toBeVisible(); + await expect(popup.locator('#authForm')).toBeHidden(); + await popup.click('#authPanel summary'); + await expect(popup.locator('#authForm')).toBeVisible(); + await expect(popup.locator('#authSignUpLink')).toHaveAttribute('href', /^https:\/\/proscanbot\.web\.app\/dashboard\//); + await expect(popup.locator('#authResetBtn')).toBeVisible(); + await expect(popup.locator('#exportToProScanBtn')).toBeHidden(); + await popup.click('#authResetBtn'); + await expect(popup.locator('#authError')).toContainText('Enter your email'); +}); + +test('two runs without an export keep their own attribution (F-20)', async ({ ext }) => { + test.skip(!Flags.CLOUD_SYNC, 'cloud sync is off in this build'); await serveAmazon(ext.context, simplePlan(2)); const store = await extPage(ext); - const s = await twoRuns(ext, store, (x) => (x.outbox || []).length >= 4); + // Signed in as far as the engine knows. Firebase has no user in this + // profile, so nothing is sent anywhere and the outbox keeps it all. Seed + // after the worker's first auth check, which clears a stale account. + await store.evaluate(() => new Promise((r) => chrome.runtime.sendMessage({ type: 'PROSCAN_AUTH_STATE' }, r))); + await store.evaluate(() => chrome.storage.local.set({ account: { uid: 'e2e-user' } })); + const s = await twoRuns(ext, store, (x) => (x.outbox || []).length >= 6); const runIds = [...new Set(s.outbox.map((p) => p.runId))]; expect(runIds).toHaveLength(2); + expect(runIds.map((id) => id.replace(/_\d+$/, '')).sort()).toEqual(['k_garden-hose', 'k_yoga-mat']); + expect(s.outbox.every((e) => e.uid === 'e2e-user')).toBe(true); + expect(s.outbox.map((e) => e.kind)).toEqual(['page', 'page', 'run', 'page', 'page', 'run']); for (const runId of runIds) { expect(runMetaFor(s, runId)).toBeTruthy(); } diff --git a/tests/e2e/upgrade.spec.mjs b/tests/e2e/upgrade.spec.mjs index 8a5ac97..6a694c1 100644 --- a/tests/e2e/upgrade.spec.mjs +++ b/tests/e2e/upgrade.spec.mjs @@ -17,7 +17,7 @@ const V21_REV = '40be432'; const V20 = path.join(BUILD_DIR, 'v2.0'); const V21 = path.join(BUILD_DIR, 'v2.1'); const SLOT = path.join(BUILD_DIR, 'upgrade-slot'); -const CURRENT_VERSION = '2.2.0'; +const CURRENT_VERSION = '2.3.0'; function bug(fid, what) { test.fail(!process.env.PROSCAN_SHOW_KNOWN, `${fid}: ${what}`); diff --git a/tests/golden/__snapshots__/characterize.test.js.snap b/tests/golden/__snapshots__/characterize.test.js.snap index bb09899..4080920 100644 --- a/tests/golden/__snapshots__/characterize.test.js.snap +++ b/tests/golden/__snapshots__/characterize.test.js.snap @@ -281,6 +281,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-lastpage 1`] = ` "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Last Page Product", "organicRank": 1, @@ -293,6 +294,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-lastpage 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$19.99", "priceCents": 1999, "rating": 4, @@ -342,6 +344,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "dReviews": null, "isNew": true, }, + "img": null, "isPrime": true, "name": "Test Widget Pro 2000", "organicRank": 1, @@ -354,6 +357,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$29.99", "priceCents": 2999, "rating": 4.5, @@ -372,6 +376,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Budget Gadget Basic", "organicRank": 2, @@ -384,6 +389,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$9.99", "priceCents": 999, "rating": 3.8, @@ -402,6 +408,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Mystery Item No Data", "organicRank": 3, @@ -414,6 +421,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "sponsored": false, }, ], + "prev": null, "price": null, "priceCents": null, "rating": null, @@ -432,6 +440,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "dReviews": null, "isNew": true, }, + "img": null, "isPrime": true, "name": "Legacy Star Product", "organicRank": 4, @@ -444,6 +453,7 @@ exports[`current scraper behavior on the corpus 2026-02/search-results 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$49.99", "priceCents": 4999, "rating": 4.2, @@ -575,6 +585,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-no-pagination-syn "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Rare Widget, single result", "organicRank": 1, @@ -587,6 +598,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-no-pagination-syn "sponsored": false, }, ], + "prev": null, "price": "$41.00", "priceCents": 4100, "rating": null, @@ -605,6 +617,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-no-pagination-syn "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Rare Widget, second result", "organicRank": 2, @@ -617,6 +630,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-no-pagination-syn "sponsored": false, }, ], + "prev": null, "price": "$1,249.00", "priceCents": 124900, "rating": null, @@ -666,6 +680,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Acme Widget Pro, 6 ft, Braided", "organicRank": 3, @@ -684,6 +699,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "sponsored": false, }, ], + "prev": null, "price": "$12.99", "priceCents": 1299, "rating": 4.4, @@ -702,6 +718,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Widget Basic, 3 ft", "organicRank": 1, @@ -714,6 +731,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "sponsored": false, }, ], + "prev": null, "price": "$7.49", "priceCents": 749, "rating": 4.1, @@ -732,6 +750,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "dReviews": null, "isNew": true, }, + "img": null, "isPrime": false, "name": "Bulk pack of widgets, 24 count", "organicRank": 2, @@ -744,6 +763,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-title-recipe-synt "sponsored": false, }, ], + "prev": null, "price": null, "priceCents": null, "rating": null, @@ -793,6 +813,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81s9TLcupfL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Thick Yoga Mat, Exercise & Fitness Mat with Easy-Cinch Carrying Strap", "organicRank": null, @@ -805,6 +826,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$21.23", "priceCents": 2123, "rating": 4.6, @@ -823,6 +845,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61JebJK2odL._AC_UL320_.jpg", "isPrime": false, "name": "Retrospec Solana Yoga Mat Thick 1/2in Non-Slip Workout Mat with Nylon Strap - 72x24in Exercise Mat for Pilates, Stretching & Fitness - BPA Free, Easy Clean", "organicRank": 10, @@ -841,6 +864,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$24.99", "priceCents": 2499, "rating": 4.4, @@ -859,6 +883,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61v1tPJufHL._AC_UL320_.jpg", "isPrime": false, "name": "Yoga Mat 72" x 24" x 6mm – Non Slip Exercise Mat", "organicRank": null, @@ -871,6 +896,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$29.99", "priceCents": 2999, "rating": 4.8, @@ -889,6 +915,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71aaK-w9tbL._AC_UL320_.jpg", "isPrime": false, "name": "Retrospec Solana Yoga Mat 1" Thick w/Nylon Strap for Men & Women - Non Slip Exercise Mat for Home Yoga, Pilates, Stretching, Floor & Fitness Workouts", "organicRank": null, @@ -901,6 +928,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$39.99", "priceCents": 3999, "rating": 4.5, @@ -919,6 +947,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61KZlPKYscL._AC_UL320_.jpg", "isPrime": false, "name": "Gruper Yoga Mat Non Slip, Eco Friendly Exercise Mat with Carrying Strap", "organicRank": 1, @@ -931,6 +960,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$25.19", "priceCents": 2519, "rating": 4.4, @@ -949,6 +979,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71Dw6U5ZNVL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Thick Yoga Mat, Exercise & Fitness Mat with Easy-Cinch Carrying Strap", "organicRank": 2, @@ -961,6 +992,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$24.98", "priceCents": 2498, "rating": 4.6, @@ -979,6 +1011,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/611L9oYa0DL._AC_UL320_.jpg", "isPrime": false, "name": "Yoga Knee Pad Cushion, 0.6 Inch (15mm) Mini Yoga Mat for Home Workout", "organicRank": 3, @@ -991,6 +1024,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$8.99", "priceCents": 899, "rating": 4.5, @@ -1009,6 +1043,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71U3oP3IqsL._AC_UL320_.jpg", "isPrime": false, "name": "Amazon Basics Extra Thick Exercise Yoga Mat with Carrying Strap", "organicRank": 4, @@ -1021,6 +1056,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": null, "priceCents": null, "rating": 4.6, @@ -1039,6 +1075,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81mao7QZ1iL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Thick Yoga Mat, Exercise & Fitness Mat with Easy-Cinch Carrying Strap", "organicRank": 5, @@ -1051,6 +1088,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$25.02", "priceCents": 2502, "rating": 4.6, @@ -1069,6 +1107,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81gN-e265yL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Thick Yoga Mat, Exercise & Fitness Mat with Easy-Cinch Carrying Strap", "organicRank": 6, @@ -1081,6 +1120,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$21.11", "priceCents": 2111, "rating": 4.6, @@ -1099,6 +1139,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71aAvY+4ZqL._AC_UL320_.jpg", "isPrime": false, "name": "HAPBEAR Foldable Yoga Mat with Towel Set - 72"x24"x0.24" (6mm), Non-Slip TPE Exercise Mat with Absorbent Yoga Towel for Home Workout, Yoga, Pilates, Stretching & Travel, Durable & Eco-Friendly, Includes Carry Bag", "organicRank": null, @@ -1111,6 +1152,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$39.99", "priceCents": 3999, "rating": 4.7, @@ -1129,6 +1171,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81chpv2CQLL._AC_UL320_.jpg", "isPrime": false, "name": "COOLMOON Yoga Mat Non Slip, Anti-Tear 1/4 Thick TPE Yoga Mats for Women and Men, 72"x24" Exercise & Fitness Mat with Carrying Strap, Workout Mats for Yoga, Pilates and Floor Exercise", "organicRank": null, @@ -1141,6 +1184,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$36.99", "priceCents": 3699, "rating": 4.2, @@ -1159,6 +1203,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/710jyzqZ06L._AC_UL320_.jpg", "isPrime": false, "name": "PAIDU Foldable Yoga Mat, 70"x24"x0.31" TPE Non-Slip Mat with Bag", "organicRank": null, @@ -1171,6 +1216,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$42.65", "priceCents": 4265, "rating": 4.4, @@ -1189,6 +1235,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61HbxvB0gXL._AC_UL320_.jpg", "isPrime": false, "name": "PAIDU Foldable Yoga Mat 10mm Thick, 75"x26" Non-Slip Fitness Exercise Mat", "organicRank": null, @@ -1201,6 +1248,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$51.49", "priceCents": 5149, "rating": 4.3, @@ -1219,6 +1267,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61wBZtl1D5L._AC_UL320_.jpg", "isPrime": false, "name": "6.2x3ft Folding Gym Mat, 3" Thick", "organicRank": 7, @@ -1231,6 +1280,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$59.99", "priceCents": 5999, "rating": 5, @@ -1249,6 +1299,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71MQ8weHJOL._AC_UL320_.jpg", "isPrime": false, "name": "Gruper Yoga Mat Non Slip, Eco Friendly Exercise Mat with Carrying Strap", "organicRank": 8, @@ -1261,6 +1312,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$28.16", "priceCents": 2816, "rating": 4.4, @@ -1279,6 +1331,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81oQh5H8mWL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Yoga Mat - Premium 6mm Print Reversible Extra Thick Non Slip Exercise & Fitness Mat for All Types of Yoga, Pilates & Floor Workouts (68" x 24" x 6mm Thick)", "organicRank": 9, @@ -1291,6 +1344,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$22.75", "priceCents": 2275, "rating": 4.6, @@ -1309,6 +1363,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/41J4tyGjZzL._AC_UL320_.jpg", "isPrime": false, "name": "Stakt Foldable Yoga Mat Pro - Non Slip 2 in 1 Fitness Exercise Mat", "organicRank": null, @@ -1321,6 +1376,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$128.00", "priceCents": 12800, "rating": 4.2, @@ -1339,6 +1395,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61evs7yyd4L._AC_UL320_.jpg", "isPrime": false, "name": "Foldable Yoga Mat Extra Large 74"×31.5"", "organicRank": null, @@ -1351,6 +1408,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$45.99", "priceCents": 4599, "rating": 4.4, @@ -1369,6 +1427,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/712NJegtSpL._AC_UL320_.jpg", "isPrime": false, "name": "Yoga Mat, 80cm Wide, 1/3 Inch Thick Exercise Mat, Carry Strap, Fitness", "organicRank": null, @@ -1381,6 +1440,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$29.99", "priceCents": 2999, "rating": 4.5, @@ -1399,6 +1459,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/614eJq9mV0L._AC_UL320_.jpg", "isPrime": false, "name": "KEEP 72" x 24" Yoga Mat, 7mm Thick Non Slip TPE Mat", "organicRank": null, @@ -1411,6 +1472,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": true, }, ], + "prev": null, "price": "$24.99", "priceCents": 2499, "rating": 4.3, @@ -1429,6 +1491,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71f4oCDw3KL._AC_UL320_.jpg", "isPrime": false, "name": "CAP Barbell 1/2-Inch High Density Exercise Yoga Mat with Strap | Multiple Options", "organicRank": 11, @@ -1441,6 +1504,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$15.99", "priceCents": 1599, "rating": 4.6, @@ -1459,6 +1523,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71qgt7f5j5L._AC_UL320_.jpg", "isPrime": false, "name": "CAP Barbell 1/2-Inch High Density Exercise Yoga Mat with Strap | Multiple Options", "organicRank": 12, @@ -1471,6 +1536,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$15.99", "priceCents": 1599, "rating": 4.6, @@ -1489,6 +1555,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61KH78eW9lL._AC_UL320_.jpg", "isPrime": false, "name": "KEEP Extra Wide Yoga Mat, 7mm Thick 72" x 32" Exercise Mat", "organicRank": 13, @@ -1501,6 +1568,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$29.98", "priceCents": 2998, "rating": 4.5, @@ -1519,6 +1587,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/713U7n-k-3L._AC_UL320_.jpg", "isPrime": false, "name": "18 Pcs EVA Foam Floor Tiles, Puzzle Exercise Mat Set for Gym", "organicRank": 14, @@ -1531,6 +1600,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$22.49", "priceCents": 2249, "rating": 4.2, @@ -1549,6 +1619,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61T8eKzs2VL._AC_UL320_.jpg", "isPrime": false, "name": "Retrospec Solana Yoga Mat 1" Thick w/Nylon Strap for Men & Women - Non Slip Exercise Mat for Home Yoga, Pilates, Stretching, Floor & Fitness Workouts", "organicRank": 15, @@ -1561,6 +1632,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$39.99", "priceCents": 3999, "rating": 4.5, @@ -1579,6 +1651,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71v1o-kvwjL._AC_UL320_.jpg", "isPrime": false, "name": "CAP Barbell Folding Exercise Mat – Durable, Anti-Tear, Thick Padding for Fitness, Aerobics, Gymnastics & Home Workouts. 72"L x 24"W x 2"Thick. BLACK", "organicRank": 16, @@ -1591,6 +1664,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$36.97", "priceCents": 3697, "rating": 4.7, @@ -1609,6 +1683,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71iqfgjEfGL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Thick Yoga Mat, Exercise & Fitness Mat with Easy-Cinch Carrying Strap", "organicRank": 17, @@ -1621,6 +1696,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$24.98", "priceCents": 2498, "rating": 4.6, @@ -1639,6 +1715,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71pBRbyl-TL._AC_UL320_.jpg", "isPrime": false, "name": "Gruper Yoga Mat Non Slip, Anti-Tear Workout Mat with Carrying Strap and Bag", "organicRank": 18, @@ -1651,6 +1728,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$22.49", "priceCents": 2249, "rating": 4.4, @@ -1669,6 +1747,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/7171WaS6I3L._AC_UL320_.jpg", "isPrime": false, "name": "Fitvids All Purpose 1/4-Inch High Density Anti-Tear Exercise Yoga Mat with Carrying Strap with Optional Yoga Blocks, Multiple Colors", "organicRank": 19, @@ -1681,6 +1760,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$16.99", "priceCents": 1699, "rating": 4.4, @@ -1699,6 +1779,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81YjZmUPryL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Yoga Mat - Premium 5mm Solid Thick Non Slip Exercise & Fitness Mat for All Types of Yoga, Pilates & Floor Workouts (68" x 24" x 5mm)", "organicRank": 20, @@ -1711,6 +1792,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$20.81", "priceCents": 2081, "rating": 4.4, @@ -1729,6 +1811,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81U56-TDp+L._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Premium Yoga Mat, 6mm Thick Print Mat", "organicRank": 21, @@ -1741,6 +1824,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$19.29", "priceCents": 1929, "rating": 4.7, @@ -1759,6 +1843,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/811FZqWCxIL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Premium Yoga Mat, 6mm Thick Print Mat", "organicRank": 22, @@ -1771,6 +1856,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$21.00", "priceCents": 2100, "rating": 4.7, @@ -1789,6 +1875,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71jk4ZHwsdL._AC_UL320_.jpg", "isPrime": false, "name": "CAP Barbell 1/2-Inch High Density Exercise Yoga Mat with Strap | Multiple Options", "organicRank": 23, @@ -1801,6 +1888,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$18.99", "priceCents": 1899, "rating": 4.6, @@ -1819,6 +1907,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61I+rnqoMcL._AC_UL320_.jpg", "isPrime": false, "name": "Retrospec Solana Yoga Mat Thick 1/2in Non-Slip Workout Mat with Nylon Strap - 72x24in Exercise Mat for Pilates, Stretching & Fitness - BPA Free, Easy Clean", "organicRank": 24, @@ -1831,6 +1920,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$24.99", "priceCents": 2499, "rating": 4.4, @@ -1849,6 +1939,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/7156TmPL+sL._AC_UL320_.jpg", "isPrime": false, "name": "KEEP Extra Thick NBR Yoga Mat for Women & Men, 10mm Thick, 72”x32” Large Exercise Mat with Non-Slip Workout Mat for Yoga, Pilates, Stretching, Meditation – Wide & Cushioned Fitness Mat", "organicRank": 25, @@ -1861,6 +1952,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$21.99", "priceCents": 2199, "rating": 4.5, @@ -1879,6 +1971,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81T1NUS+UwL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Yoga Mat Classic Print, 4mm Non-Slip Mat", "organicRank": 26, @@ -1891,6 +1984,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$24.38", "priceCents": 2438, "rating": 4.5, @@ -1909,6 +2003,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/91WnKjpclwL._AC_UL320_.jpg", "isPrime": false, "name": "Manduka PRO Yoga Mat - 6mm | Lifetime Durability | Hygienic Construction | Premium Studio Quality | Teacher Approved", "organicRank": 27, @@ -1921,6 +2016,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$144.00", "priceCents": 14400, "rating": 4.5, @@ -1939,6 +2035,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/819cPiAoMuL._AC_UL320_.jpg", "isPrime": false, "name": "YOGATI® Yoga Mat with Strap with Alignment Lines. Home Workout Mat for Women, Men and Kids. Thick Non Slip Yoga Mat for Pilates and Fitness. Brown, Pink and Purple Yoga Mats", "organicRank": 28, @@ -1951,6 +2048,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$19.98", "priceCents": 1998, "rating": 4.5, @@ -1969,6 +2067,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/711NCt1h9zL._AC_UL320_.jpg", "isPrime": false, "name": "Voyage 5mm Studio Yoga Mats, 6 Pack, 72x24in", "organicRank": 29, @@ -1981,6 +2080,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$99.99", "priceCents": 9999, "rating": 4.8, @@ -1999,6 +2099,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81YdT2ovoPL._AC_UL320_.jpg", "isPrime": false, "name": "Economy 3mm Bulk Yoga Mats, 10 Pack, 68x24in", "organicRank": 30, @@ -2011,6 +2112,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$89.99", "priceCents": 8999, "rating": 4.7, @@ -2029,6 +2131,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/91hz7hERYjS._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Yoga Mat - Premium 5mm Solid Thick Non Slip Exercise & Fitness Mat for All Types of Yoga, Pilates & Floor Workouts (68" x 24" x 5mm)", "organicRank": 31, @@ -2041,6 +2144,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$22.65", "priceCents": 2265, "rating": 4.4, @@ -2059,6 +2163,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/91UdCyPWQLL._AC_UL320_.jpg", "isPrime": false, "name": "Manduka PROlite Yoga Mat 4.7mm | Lifetime Durability | Hygienic Construction | Teacher Approved", "organicRank": 32, @@ -2071,6 +2176,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$99.99", "priceCents": 9999, "rating": 4.5, @@ -2089,6 +2195,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61WrjbRYC3L._AC_UL320_.jpg", "isPrime": false, "name": "Amazon Basics Extra Thick Exercise Yoga Mat with Carrying Strap", "organicRank": 33, @@ -2101,6 +2208,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": null, "priceCents": null, "rating": 4.6, @@ -2119,6 +2227,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71WJvgWQy-L._AC_UL320_.jpg", "isPrime": false, "name": "Fitvids 1/2-inch Thick High Density Exercise Yoga Mat with Carrying Strap", "organicRank": 34, @@ -2131,6 +2240,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$20.99", "priceCents": 2099, "rating": 4.5, @@ -2149,6 +2259,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/61rFL8O9XZS._AC_UL320_.jpg", "isPrime": false, "name": "Manduka PRO Yoga Mat - 6mm | Lifetime Durability | Hygienic Construction | Premium Studio Quality | Teacher Approved", "organicRank": 35, @@ -2161,6 +2272,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$144.00", "priceCents": 14400, "rating": 4.5, @@ -2179,6 +2291,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81KHLWfRt5L._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Essentials 1/4" Thin (6mm) Yoga & Pilates, Fitness & Exercise Mat with Easy-Cinch Carrier Strap Cusion Support For Fitness and Gym Workouts", "organicRank": 36, @@ -2191,6 +2304,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$20.69", "priceCents": 2069, "rating": 4.6, @@ -2209,6 +2323,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81PfuUvfvkL._AC_UL320_.jpg", "isPrime": false, "name": "Funtery 3 mm Thin Yoga Mats Bulk, 68 x 24 in", "organicRank": 37, @@ -2221,6 +2336,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$129.99", "priceCents": 12999, "rating": 3.6, @@ -2239,6 +2355,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71+n2fvgNoL._AC_UL320_.jpg", "isPrime": false, "name": "Sunny Health & Fitness Exercise Yoga Mats - Non-Slip Support for Home & Gym", "organicRank": 38, @@ -2251,6 +2368,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$18.99", "priceCents": 1899, "rating": 4.4, @@ -2269,6 +2387,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/91TL-Sk5aNL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Premium Yoga Mat, 6mm Thick Print Mat", "organicRank": 39, @@ -2281,6 +2400,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$39.99", "priceCents": 3999, "rating": 4.7, @@ -2299,6 +2419,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71S+4POpCiL._AC_UL320_.jpg", "isPrime": false, "name": "Cork Yoga Mat with Natural Rubber Base, Extra Size, Thickness and Support, Excellent Cushion & Grip, Non-Slip, Non-Toxic, Sweat-Resistant, Sustainable, Eco-friendly Exercise Mat", "organicRank": 40, @@ -2311,6 +2432,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$89.00", "priceCents": 8900, "rating": 4.6, @@ -2329,6 +2451,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81xWBR2159L._AC_UL320_.jpg", "isPrime": false, "name": "SPRI Hanging Exercise Mats for Gyms, Closed-Cell Foam Yoga & Fitness Mats with Grommets, Thick Workout Mats for Studios & Group Fitness Classes", "organicRank": 41, @@ -2341,6 +2464,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$233.01", "priceCents": 23301, "rating": 4.7, @@ -2359,6 +2483,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71EhT3gPpNL._AC_UL320_.jpg", "isPrime": false, "name": "Amazon Basics 1/4 Inch Thick TPE Exercise Yoga Mat with Carrying Strap", "organicRank": 42, @@ -2371,6 +2496,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$19.27", "priceCents": 1927, "rating": 4.6, @@ -2389,6 +2515,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71zC5MVG2xL._AC_UL320_.jpg", "isPrime": false, "name": "CAMBIVO Extra Long Wide Yoga Mat, Thick PVC Workout Mat", "organicRank": 43, @@ -2401,6 +2528,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$39.99", "priceCents": 3999, "rating": 4.6, @@ -2419,6 +2547,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81P7dfAuglL._AC_UL320_.jpg", "isPrime": false, "name": "Gaiam Yoga Mat - Premium 5mm Solid Thick Non Slip Exercise & Fitness Mat for All Types of Yoga, Pilates & Floor Workouts (68" x 24" x 5mm)", "organicRank": 44, @@ -2431,6 +2560,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$23.47", "priceCents": 2347, "rating": 4.4, @@ -2449,6 +2579,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81PK-dlA1aL._AC_UL320_.jpg", "isPrime": false, "name": "Heathyoga Yoga Mat Non Slip TPE Pilates Exercise Mats for Home Workout Gym Fitness Eco Friendly Thick Hot Yoga Mat with Strap Body Alignment,72"x 26" Thickness 1/4"", "organicRank": 45, @@ -2461,6 +2592,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$36.99", "priceCents": 3699, "rating": 4.5, @@ -2479,6 +2611,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/71YPoM9x53L._AC_UL320_.jpg", "isPrime": false, "name": "ProsourceFit Exercise Balance Pad, Non-Slip Cushioned Foam Mat & Knee Pad for Fitness and Stability Training, Yoga, Physical Therapy", "organicRank": 46, @@ -2491,6 +2624,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$26.99", "priceCents": 2699, "rating": 4.8, @@ -2509,6 +2643,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/81ZKYeFRa8L._AC_UL320_.jpg", "isPrime": false, "name": "CAP Non-Slip Yoga Mat for Home Workout & Pilates, Lightweight Exercise Fitness Mat with Textured Grip Surface, Portable Roll-Up Yoga Mat for Women & Men", "organicRank": 47, @@ -2521,6 +2656,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$11.99", "priceCents": 1199, "rating": 4.2, @@ -2539,6 +2675,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "dReviews": null, "isNew": true, }, + "img": "https://m.media-amazon.com/images/I/51M7L1s50VL._AC_UL320_.jpg", "isPrime": false, "name": "Liforme Travel Yoga Mat - Patented Alignment Design, Advanced Non-Slip Grip", "organicRank": 48, @@ -2551,6 +2688,7 @@ exports[`current scraper behavior on the corpus 2026-09/search-yoga-mat 1`] = ` "sponsored": false, }, ], + "prev": null, "price": "$160.00", "priceCents": 16000, "rating": 4.4, diff --git a/tests/setup/esm-to-cjs-transform.js b/tests/setup/esm-to-cjs-transform.js index 582d0eb..1ccc5f3 100644 --- a/tests/setup/esm-to-cjs-transform.js +++ b/tests/setup/esm-to-cjs-transform.js @@ -12,9 +12,11 @@ * * Supported forms (all that sync.js uses): * import { a, b } from 'mod'; -> const { a, b } = require('mod'); + * import { a as b } from 'mod'; -> const { a: b } = require('mod'); * import x from 'mod'; -> const x = require('mod'); * export async function f() {} -> async function f() {}; module.exports.f = f; * export function f() {} -> function f() {}; module.exports.f = f; + * export const X = ...; -> const X = ...; module.exports.X = X; */ function rewrite(src) { const named = []; @@ -26,7 +28,7 @@ function rewrite(src) { (_m, names, mod) => { const clean = names .split(',') - .map((s) => s.trim()) + .map((s) => s.trim().replace(/\s+as\s+/, ': ')) .filter(Boolean) .join(', '); return `const { ${clean} } = require('${mod}');`; @@ -37,6 +39,11 @@ function rewrite(src) { /import\s+([A-Za-z_$][\w$]*)\s+from\s*['"]([^'"]+)['"]\s*;?/g, (_m, name, mod) => `const ${name} = require('${mod}');` ) + // export const NAME = ... -> strip `export`, remember the name + .replace(/^export\s+const\s+([A-Za-z_$][\w$]*)/gm, (_m, name) => { + named.push(name); + return `const ${name}`; + }) // export (async) function name(...) -> strip `export`, remember the name .replace( /export\s+(async\s+)?function\s+([A-Za-z_$][\w$]*)/g, @@ -52,8 +59,15 @@ function rewrite(src) { return out; } +const crypto = require('crypto'); +const self = require('fs').readFileSync(__filename); + module.exports = { process(sourceText) { return { code: rewrite(sourceText) }; }, + // Jest caches transformed files; a change here must invalidate them. + getCacheKey(sourceText, filename) { + return crypto.createHash('md5').update(self).update(sourceText).update(filename).digest('hex'); + }, }; diff --git a/tests/unit/engine.test.js b/tests/unit/engine.test.js index 82f41a3..1c10dcd 100644 --- a/tests/unit/engine.test.js +++ b/tests/unit/engine.test.js @@ -310,6 +310,18 @@ describe('the worker stopped between pages', () => { expect(st.run).toMatchObject({ state: 'failed', reason: 'interrupted' }); expect(st.results).toHaveLength(4); }); + + test('a run ended after a restart queues its final header for the signed-in account', async () => { + const rig = createRig({ site: simpleSite(3), flags: { CLOUD_SYNC: true }, local: { account: { uid: 'u1' } } }); + await startIn(rig, 'restart'); + await settle(); + await rig.session.remove('run'); + rig.killWorker(); + await rig.engine().recover(); + const db = await rig.db(); + expect((await db.getAll('outbox')).map((e) => e.kind)).toEqual(['page', 'run']); + db.close(); + }); }); describe('a page that never reports', () => { @@ -439,8 +451,8 @@ describe('products across pages (F-27, F-28)', () => { const PAGE2 = page(card('B0C', '$3.00') + card('B0A', '$5.00') + card('B0B', '$2.00', { rating: false }), null); const site = (url) => (pageNo(url) === 1 ? PAGE1 : PAGE2); - async function twoPages({ lastValues = [], flags } = {}) { - const rig = createRig({ site, flags }); + async function twoPages({ lastValues = [], flags, local } = {}) { + const rig = createRig({ site, flags, local }); const db = await rig.db(); await db.write(lastValues.map((v) => ({ store: 'lastValues', put: v }))); db.close(); @@ -487,15 +499,50 @@ describe('products across pages (F-27, F-28)', () => { after.close(); }); - test('with cloud sync on each page is queued once, and the bundle has the products as they are now', async () => { + test('with cloud sync on but nobody signed in nothing is queued (F-26)', async () => { const rig = await twoPages({ flags: { CLOUD_SYNC: true } }); - const { bundle, seqs } = await rig.engine().outboxBundle(); - expect(seqs).toHaveLength(2); - expect(bundle.syncQueue.map((p) => p.asin)).toEqual(['B0A', 'B0B', 'B0C']); - expect(bundle.syncQueue[0].placements).toHaveLength(3); - expect(bundle.scrapeRunMeta).toMatchObject({ keyword: 'w' }); - await rig.engine().clearOutbox(seqs); - expect((await rig.engine().outboxBundle()).seqs).toEqual([]); + const db = await rig.db(); + expect(await db.count('outbox')).toBe(0); + db.close(); + }); + + test('signed in, each page is queued once for that account, then the run (F-20, F-29e)', async () => { + const rig = await twoPages({ flags: { CLOUD_SYNC: true }, local: { account: { uid: 'u1', email: 'a@b.c' } } }); + const run = await rig.run(); + const db = await rig.db(); + const entries = await db.getAll('outbox'); + expect(entries.map((e) => [e.kind, e.pageIndex, e.uid, e.runId])).toEqual([ + ['page', 1, 'u1', run.runId], + ['page', 2, 'u1', run.runId], + ['run', undefined, 'u1', run.runId], + ]); + db.close(); + }); + + test('the run id is minted once as {sourceId}_{startMs}, with the local day key (F-20, F-29d)', async () => { + const rig = await twoPages(); + const run = await rig.run(); + expect(run.runId).toBe(`k_w_${run.startedAt}`); + expect(run.sourceId).toBe('k_w'); + expect(run.source).toMatchObject({ type: 'keyword', keyword: 'w', sourceId: 'k_w' }); + expect(run.dayKey).toMatch(/^\d{4}-\d{2}-\d{2}$/); + const db = await rig.db(); + expect((await db.runPages(run.runId)).map((p) => p.pageIndex)).toEqual([1, 2]); + db.close(); + }); + + test("another account's last values give no delta, and new ones carry the uid (F-29e)", async () => { + const theirs = { ...prevRun('B0A', 600), uid: 'someone-else' }; + const anon = prevRun('B0B', 250); + const rig = await twoPages({ lastValues: [theirs, anon], local: { account: { uid: 'u1' } } }); + const [a, b] = await results(rig); + expect(a.delta).toMatchObject({ isNew: true }); + expect(a.prev).toBeNull(); + expect(b.delta).toMatchObject({ isNew: false, dPriceCents: -50 }); + expect(b.prev).toMatchObject({ priceCents: 250, rating: 4.5, reviewCount: 900, uid: null }); + const db = await rig.db(); + expect(await db.get('lastValues', 'B0A')).toMatchObject({ priceCents: 500, uid: 'u1' }); + db.close(); }); test('each page drops lastValues older than the age limit (F-26)', async () => { @@ -658,3 +705,30 @@ describe('old runs', () => { db.close(); }); }); + +describe('the end of a synced run (EXT9-5)', () => { + test('signed out mid-run, the final header still goes to the account its pages went to', async () => { + const rig = createRig({ site: simpleSite(3), flags: { CLOUD_SYNC: true }, local: { account: { uid: 'u1' } } }); + await startIn(rig, 'leave'); + await settle(); + await rig.local.remove('account'); + expect(await rig.popup({ type: 'STOP_RUN' })).toMatchObject({ stopped: true }); + const db = await rig.db(); + const outbox = await db.getAll('outbox'); + db.close(); + expect(outbox.map((e) => [e.kind, e.uid])).toEqual([['page', 'u1'], ['run', 'u1']]); + }); + + test('a run left live only in IndexedDB queues its final header when it is ended', async () => { + const rig = createRig({ site: simpleSite(3), flags: { CLOUD_SYNC: true }, local: { account: { uid: 'u1' } } }); + await startIn(rig, 'gone'); + await settle(1000); + await rig.session.remove('run'); + rig.killWorker(); + await rig.popup({ type: 'GET_STATE' }); + const db = await rig.db(); + const kinds = (await db.getAll('outbox')).map((e) => e.kind); + db.close(); + expect(kinds).toContain('run'); + }); +}); diff --git a/tests/unit/parsers.test.js b/tests/unit/parsers.test.js index da52ea3..20fca4c 100644 --- a/tests/unit/parsers.test.js +++ b/tests/unit/parsers.test.js @@ -184,7 +184,15 @@ describe('Parsers.scrapeProduct fields', () => { test('a card with no title, rating or reviews has nulls, not 0 or N/A', () => { const p = Parsers.scrapeProduct(one('
')); - expect(p).toMatchObject({ name: null, price: null, priceCents: null, rating: null, reviewCount: null }); + expect(p).toMatchObject({ name: null, price: null, priceCents: null, rating: null, reviewCount: null, img: null }); + }); + + test('the image comes from Amazon\'s image host only (F-25)', () => { + const img = (src) => `
`; + const good = 'https://m.media-amazon.com/images/I/81s9TLcupfL._AC_UL320_.jpg'; + expect(Parsers.scrapeProduct(one(img(good))).img).toBe(good); + expect(Parsers.scrapeProduct(one(img('data:image/gif;base64,R0lGOD'))).img).toBeNull(); + expect(Parsers.scrapeProduct(one(img('https://evil.example/x.jpg'))).img).toBeNull(); }); }); diff --git a/tests/unit/schema.test.js b/tests/unit/schema.test.js new file mode 100644 index 0000000..41c0a87 --- /dev/null +++ b/tests/unit/schema.test.js @@ -0,0 +1,130 @@ +/** + * @jest-environment node + * + * The shared cloud schema: id builders, points, deltas and validators. + */ +const S = require('../../packages/schema/index.js'); + +describe('source ids (F-29d)', () => { + const idOf = (url) => S.sourceIdOf(S.sourceOf(url)); + + test('a storefront is s_{seller}, from me=, seller= or rh', () => { + expect(idOf('https://www.amazon.com/s?me=A3K9XELT4QZ6M2&marketplaceID=ATVPDKIKX0DER')).toBe('s_A3K9XELT4QZ6M2'); + expect(idOf('https://www.amazon.com/s?seller=a3k9xelt4qz6m2')).toBe('s_A3K9XELT4QZ6M2'); + expect(idOf('https://www.amazon.com/s?i=merchant-items&rh=p_6%3AA3K9XELT4QZ6M2')).toBe('s_A3K9XELT4QZ6M2'); + expect(idOf('https://www.amazon.com/s?rh=n%3A1055398%2Cme%3AA3K9XELT4QZ6M2')).toBe('s_A3K9XELT4QZ6M2'); + expect(S.sourceOf('https://www.amazon.com/s?me=A3K9XELT4QZ6M2&k=mug')).toMatchObject({ type: 'storefront', keyword: 'mug' }); + }); + + test('a keyword is k_{slug}, the same for the same search', () => { + expect(idOf('https://www.amazon.com/s?k=Yoga+Mat&page=2')).toBe('k_yoga-mat'); + expect(idOf('https://www.amazon.com/s?k=yoga%20mat&ref=sr_pg_3')).toBe('k_yoga-mat'); + }); + + test('non-ASCII and long keywords get a hash, so they never collide', () => { + const a = idOf('https://www.amazon.com/s?k=%E6%9D%AF%E5%AD%90'); + const b = idOf('https://www.amazon.com/s?k=%E7%A2%97'); + expect(a).toMatch(/^k_h[0-9a-z]+$/); + expect(a).not.toBe(b); + expect(idOf('https://www.amazon.com/s?k=caf%C3%A9+mug')).toMatch(/^k_caf-mug-h[0-9a-z]+$/); + const long = idOf('https://www.amazon.com/s?k=' + 'a'.repeat(90)); + expect(long.length).toBeLessThan(80); + expect(long).not.toBe(idOf('https://www.amazon.com/s?k=' + 'a'.repeat(91))); + }); + + test('a search with neither gets an id from its query, not a shared k_unknown', () => { + const a = idOf('https://www.amazon.com/s?i=kitchen&rh=n%3A289814&page=2&qid=1'); + expect(a).toMatch(/^k_x[0-9a-z]+$/); + expect(a).toBe(idOf('https://www.amazon.com/s?rh=n%3A289814&i=kitchen&qid=9')); + expect(a).not.toBe(idOf('https://www.amazon.com/s?i=kitchen&rh=n%3A289815')); + expect(S.sourceIdOf(S.sourceOf('not a url'))).toBe('k_unknown'); + }); + + test('run and page ids', () => { + expect(S.runIdOf('k_mug', 1749477731000)).toBe('k_mug_1749477731000'); + expect(S.pageIdOf(3)).toBe('p0003'); + }); +}); + +test('the day key is the local date of the scan', () => { + const ms = Date.parse('2026-06-10T02:30:00Z'); + expect(S.dayKeyOf(ms, 0)).toBe('2026-06-10'); + // 10:30 pm in New York (UTC-4) is still June 9 there + expect(S.dayKeyOf(ms, 240)).toBe('2026-06-09'); +}); + +test('run status from the extension state', () => { + expect(S.runStatusOf('running')).toBe('active'); + expect(S.runStatusOf('done')).toBe('complete'); + expect(S.runStatusOf('stopped')).toBe('stopped'); + expect(S.runStatusOf('blocked')).toBe('stopped'); + expect(S.runStatusOf('failed')).toBe('dead'); +}); + +describe('points and deltas', () => { + test('unknown values are left out, never 0 (F-28)', () => { + expect(S.pointOf({ priceCents: null, rating: null, reviewCount: null, isPrime: false, organicRank: null })).toEqual({ pr: 0 }); + expect(S.pointOf({ priceCents: 1999, rating: 4.56, reviewCount: 0, isPrime: true, organicRank: 3 })) + .toEqual({ p: 1999, r: 4.6, v: 0, pr: 1, rk: 3 }); + }); + + test('a delta compares only what both sides know, and is null otherwise (F-21)', () => { + expect(S.deltaOf({ p: 900, r: 4.5 }, { p: 1000, r: 4.4, v: 10 }, 7)).toEqual({ p: -100, pPct: -10, r: 0.1, days: 7 }); + expect(S.deltaOf({ r: 4.5 }, { p: 1000 })).toBeNull(); + expect(S.deltaOf({ p: 900 }, null)).toBeNull(); + expect(S.deltaOf({ p: 900 }, { p: 0 })).toEqual({ p: 900 }); + }); + + test('time from numbers, dates, ISO strings and Timestamps', () => { + expect(S.timeMs(5)).toBe(5); + expect(S.timeMs(new Date(7))).toBe(7); + expect(S.timeMs('1970-01-01T00:00:00.009Z')).toBe(9); + expect(S.timeMs({ toMillis: () => 11 })).toBe(11); + expect(S.timeMs('nope')).toBeNull(); + }); +}); + +describe('validators', () => { + const run = { + sv: S.SV, runId: 'k_mug_1', sourceId: 'k_mug', + source: { type: 'keyword', sellerId: null, keyword: 'mug', url: 'https://www.amazon.com/s?k=mug' }, + mk: 'US', dayKey: '2026-06-09', startedAt: 1, finishedAt: null, status: 'active', reason: null, + pagesDone: 1, maxPages: 20, pagesPlanned: null, totalResultsOnSerp: 412, + counters: { placements: 3, uniqueAsins: 2, sponsored: 1, priceParseFailures: 0, newSeen: 2 }, + }; + const product = { + sv: S.SV, asin: 'B0C8XL4N2P', mk: 'US', name: 'Mug', url: 'https://www.amazon.com/dp/B0C8XL4N2P', img: null, + latest: { p: 2399, pr: 1, at: 5, runId: 'k_mug_1', dayKey: '2026-06-09' }, + prev: null, delta: null, sourceIds: ['k_mug'], + }; + + test('good documents pass', () => { + expect(S.validateRun(run)).toEqual([]); + expect(S.validateProduct(product)).toEqual([]); + expect(S.validateSource({ sv: S.SV, sourceId: 'k_mug', type: 'keyword', sellerId: null, keyword: 'mug', url: null, lastRunId: 'k_mug_1', lastScrapedAt: 1 })).toEqual([]); + expect(S.validatePage({ sv: S.SV, runId: 'k_mug_1', page: 1, scrapedAt: 1, expireAt: 2, count: 1, placements: 1, kind: 'last', items: { B0C8XL4N2P: { p: 1, sp: 0 } } })).toEqual([]); + expect(S.validateHistory({ sv: S.SV, asin: 'B0C8XL4N2P', d: { '2026-06-09': { p: 1 } } })).toEqual([]); + }); + + test('every document carries sv', () => { + expect(S.validateRun({ ...run, sv: undefined })).toContain(`sv: expected ${S.SV}`); + }); + + test('money must be whole cents and unknowns must be left out', () => { + const bad = { ...product, latest: { ...product.latest, p: 23.99, r: null } }; + const errs = S.validateProduct(bad); + expect(errs).toContain('latest.p: cents must be a whole number'); + expect(errs).toContain('latest.r: unknown values are left out, not null'); + expect(() => S.assertValid('product', bad)).toThrow(/invalid product document/); + }); + + test('the ASIN, status and the page size limit are checked', () => { + expect(S.validateProduct({ ...product, asin: 'b0-bad' })).toContain('asin: 10 letters and digits'); + expect(S.validateRun({ ...run, status: 'blocked' })[0]).toMatch(/^status/); + expect(S.validateRun({ ...run, finishedAt: 9 })).toContain('finishedAt: null while active'); + const items = {}; + for (let i = 0; i <= S.MAX_PAGE_ITEMS; i++) items[`B${String(i).padStart(9, '0')}`] = { p: 1 }; + expect(S.validatePage({ sv: S.SV, runId: 'r', page: 1, scrapedAt: 1, expireAt: 2, count: 0, placements: 0, items })) + .toContain(`items: at most ${S.MAX_PAGE_ITEMS}`); + }); +}); diff --git a/tests/unit/service-worker.test.js b/tests/unit/service-worker.test.js index 43a15d7..4de1b4a 100644 --- a/tests/unit/service-worker.test.js +++ b/tests/unit/service-worker.test.js @@ -8,15 +8,27 @@ jest.mock('../../scripts/background/firebase-init.js', () => ({ auth: { currentUser: { uid: 'u1', email: 'u1@example.test', displayName: null } }, db: {}, })); -jest.mock('firebase/auth/web-extension', () => ({ - onAuthStateChanged: (auth, cb) => { cb(auth.currentUser); return () => {}; }, - signInWithEmailAndPassword: async () => ({}), - signOut: async () => {}, -}), { virtual: true }); -jest.mock('../../scripts/background/sync.js', () => ({ syncToCloud: jest.fn(async () => ({ written: 1 })) })); +jest.mock('firebase/auth/web-extension', () => { + const listeners = []; + return { + __listeners: listeners, + onAuthStateChanged: (auth, cb) => { + listeners.push(cb); + cb(auth.currentUser); + return () => {}; + }, + signInWithEmailAndPassword: async () => ({}), + sendPasswordResetEmail: async () => {}, + signOut: async () => {}, + }; +}, { virtual: true }); +jest.mock('../../scripts/background/sync.js', () => { + const flush = jest.fn(async () => ({ entries: 0, pages: 0, runs: 0, products: 0, writes: 0 })); + return { createSync: () => ({ flush, pending: async () => 0, failed: async () => 0 }), isAuthError: () => false, __flush: flush }; +}); require('fake-indexeddb/auto'); -const { syncToCloud } = require('../../scripts/background/sync.js'); +const { __flush: flush } = require('../../scripts/background/sync.js'); const DB = require('../../scripts/background/db'); const fs = require('fs'); @@ -26,12 +38,14 @@ const V20 = JSON.parse(fs.readFileSync(path.join(__dirname, '../fixtures/v2.0-st const messageListeners = []; const installedListeners = []; const startupListeners = []; +const tabUpdatedListeners = []; beforeAll(() => { chrome.runtime.id = 'test-extension'; chrome.runtime.onMessage.addListener = (fn) => messageListeners.push(fn); chrome.runtime.onInstalled = { addListener: (fn) => installedListeners.push(fn) }; chrome.runtime.onStartup = { addListener: (fn) => startupListeners.push(fn) }; + chrome.tabs.onUpdated = { addListener: (fn) => tabUpdatedListeners.push(fn) }; require('../../scripts/background/service-worker.js'); }); @@ -55,10 +69,23 @@ test('one router answers every message', () => { expect(messageListeners).toHaveLength(1); }); -test('Export to ProScan is refused while cloud sync is off', async () => { +test("Export to ProScan flushes the signed-in account's outbox", async () => { + flush.mockClear(); const resp = await send({ type: 'PROSCAN_EXPORT' }); - expect(resp.error).toMatch(/not available/); - expect(syncToCloud).not.toHaveBeenCalled(); + expect(resp).toMatchObject({ ok: true, entries: 0 }); + // Export also retries entries the rules refused before. + expect(flush).toHaveBeenCalledWith('u1', { retryFailed: true }); +}); + +test('the auth state says who is signed in and what waits to sync', async () => { + const st = await send({ type: 'PROSCAN_AUTH_STATE' }); + expect(st).toMatchObject({ user: { uid: 'u1' }, notice: null, pending: 0, failed: 0, dashboardUrl: expect.stringMatching(/^https:/) }); +}); + +test('a password reset answers the same whether or not the account exists (F-56)', async () => { + const st = await send({ type: 'PROSCAN_RESET_PASSWORD', email: 'nobody@example.test' }); + expect(st).toEqual({ ok: true, message: 'If an account exists for nobody@example.test, a reset link is on its way.' }); + expect(await send({ type: 'PROSCAN_RESET_PASSWORD', email: '' })).toMatchObject({ error: expect.any(String) }); }); test('messages meant for the popup or a tab are left alone', async () => { @@ -96,3 +123,66 @@ test('the chat reads the key from settings and the run from IndexedDB', async () const st = await send({ type: 'CHAT_STATUS' }, { id: 'test-extension', tab: { id: 3 } }); expect(st).toMatchObject({ hasKey: true }); }); + +test('a session Firebase dropped without a sign-out shows as expired (F-57)', async () => { + const { auth } = require('../../scripts/background/firebase-init.js'); + const authMod = require('firebase/auth/web-extension'); + const user = auth.currentUser; + await chrome.storage.local.set({ account: { uid: 'u1', email: 'u1@example.test' } }); + auth.currentUser = null; + try { + // The worker's own listener is the first one registered. + authMod.__listeners[0](null); + await settle(); + expect(chrome.storage.local._getStore()).toMatchObject({ authNotice: 'expired' }); + expect(chrome.storage.local._getStore()).not.toHaveProperty('account'); + const st = await send({ type: 'PROSCAN_AUTH_STATE' }); + expect(st).toMatchObject({ user: null, notice: 'expired' }); + } finally { + auth.currentUser = user; + } +}); + +test('signing out on purpose clears the account without an expired notice', async () => { + await chrome.storage.local.set({ account: { uid: 'u1' }, authNotice: 'expired' }); + expect(await send({ type: 'PROSCAN_SIGN_OUT' })).toEqual({ ok: true }); + expect(chrome.storage.local._getStore()).not.toHaveProperty('account'); + expect(chrome.storage.local._getStore()).not.toHaveProperty('authNotice'); +}); + +describe('automatic flushes (EXT9-2)', () => { + const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + + test('a page load in some tab with no run does not flush', async () => { + // Let flushes asked for before this test go out first. + await wait(3200); + await settle(); + flush.mockClear(); + tabUpdatedListeners.forEach((fn) => fn(42, { status: 'loading', url: 'https://www.amazon.com/s?k=x' }, { id: 42 })); + await settle(); + await wait(3200); + expect(flush).not.toHaveBeenCalled(); + }, 15000); + + test('after a failed flush the automatic one waits, and Export does not', async () => { + await chrome.storage.local.set({ lastSync: { at: Date.now(), error: 'unavailable', failures: 2 } }); + flush.mockClear(); + // Opening the popup asks for a flush right away. + await send({ type: 'PROSCAN_AUTH_STATE' }); + await wait(50); + await settle(); + expect(flush).not.toHaveBeenCalled(); + await send({ type: 'PROSCAN_EXPORT' }); + expect(flush).toHaveBeenCalledWith('u1', { retryFailed: true }); + expect(chrome.storage.local._getStore().lastSync).toMatchObject({ error: null, failures: 0 }); + }); + + test('without a recent failure the popup flush goes out', async () => { + await chrome.storage.local.set({ lastSync: { at: Date.now() - 2 * 60 * 60 * 1000, error: 'unavailable', failures: 9 } }); + flush.mockClear(); + await send({ type: 'PROSCAN_AUTH_STATE' }); + await wait(50); + await settle(); + expect(flush).toHaveBeenCalledWith('u1', { retryFailed: false }); + }); +}); diff --git a/tests/unit/sync-plan.test.js b/tests/unit/sync-plan.test.js new file mode 100644 index 0000000..884c2f3 --- /dev/null +++ b/tests/unit/sync-plan.test.js @@ -0,0 +1,207 @@ +/** + * @jest-environment node + * + * The pure sync plan: outbox entry and IndexedDB records in, cloud writes out. + */ +const { planEntry, firstSeenCandidates, runKeys, checkRuleCaps, RULE_CAPS, DELETE } = require('../../scripts/background/sync-plan.js'); +const S = require('../../packages/schema/index.js'); + +const START = Date.parse('2026-06-09T14:02:11Z'); +const RUN_ID = `k_mug_${START}`; + +const run = (patch = {}) => ({ + runId: RUN_ID, sourceId: 'k_mug', dayKey: '2026-06-09', state: 'running', reason: null, + source: { type: 'keyword', sellerId: null, keyword: 'mug', url: 'https://www.amazon.com/s?k=mug', sourceId: 'k_mug' }, + startedAt: START, finishedAt: null, page: 2, maxPages: 20, itemCount: 3, + ...patch, +}); + +const product = (asin, patch = {}) => ({ + asin, name: `Mug ${asin}`, priceCents: 1999, rating: 4.5, reviewCount: 100, isPrime: true, + sponsored: false, organicRank: 1, img: null, runId: RUN_ID, pageIndex: 1, + scrapedAt: '2026-06-09T14:03:00.000Z', + placements: [{ page: 1, position: 1, sponsored: false, rank: 1 }], + delta: { isNew: true, dPriceCents: null, dRating: null, dReviews: null }, prev: null, + ...patch, +}); + +const pages = [ + { runId: RUN_ID, pageIndex: 1, count: 2, placements: 3, kind: 'results', scrapedAt: '2026-06-09T14:03:00.000Z', total: 412 }, + { runId: RUN_ID, pageIndex: 2, count: 1, placements: 2, kind: 'last', scrapedAt: '2026-06-09T14:03:05.000Z', total: 412 }, +]; + +const products = [ + product('B0AAAAAAA1', { + sponsored: true, + placements: [ + { page: 1, position: 1, sponsored: true, rank: null }, + { page: 2, position: 2, sponsored: false, rank: 3 }, + ], + organicRank: 3, + }), + product('B0AAAAAAA2', { + priceCents: null, organicRank: 1, placements: [{ page: 1, position: 2, sponsored: false, rank: 1 }], + delta: { isNew: false, dPriceCents: null, dRating: 0, dReviews: 5 }, + prev: { priceCents: 2100, rating: 4.5, reviewCount: 95, scrapedAt: '2026-06-02T14:00:00.000Z', uid: 'u1' }, + }), + product('B0AAAAAAA3', { pageIndex: 2, organicRank: 2, placements: [{ page: 2, position: 1, sponsored: false, rank: 2 }] }), +]; + +const entry = (patch) => ({ seq: 1, runId: RUN_ID, uid: 'u1', ...patch }); +const byPath = (writes) => Object.fromEntries(writes.map((w) => [w.path.join('/'), w])); +/** A merge write's data without the deletes that clear stale keys. */ +const kept = (data) => Object.fromEntries(Object.entries(data) + .filter(([, v]) => v !== DELETE) + .map(([k, v]) => [k, v && typeof v === 'object' && !Array.isArray(v) && Object.values(v).includes(DELETE) + ? Object.fromEntries(Object.entries(v).filter(([, x]) => x !== DELETE)) : v])); + +describe('a page entry', () => { + const writes = planEntry({ entry: entry({ kind: 'page', pageIndex: 1 }), run: run(), products, pages }); + const w = byPath(writes); + + test('writes the chunk, each ASIN on the page twice, the run and the source', () => { + expect(writes.map((x) => x.path.slice(2).join('/'))).toEqual([ + `runs/${RUN_ID}/pages/p0001`, + 'products/B0AAAAAAA1', 'products/B0AAAAAAA1/history/daily', + 'products/B0AAAAAAA2', 'products/B0AAAAAAA2/history/daily', + `runs/${RUN_ID}`, + 'sources/k_mug', + ]); + expect(writes.every((x) => x.path[0] === 'workspaces' && x.path[1] === 'u1')).toBe(true); + }); + + test('the chunk keeps every ASIN on the page with its sponsored flag and rank', () => { + const chunk = w[`workspaces/u1/runs/${RUN_ID}/pages/p0001`]; + expect(chunk.fields).toBeNull(); + expect(chunk.data).toMatchObject({ sv: S.SV, page: 1, count: 2, placements: 3, kind: 'results', total: 412 }); + expect(chunk.data.items).toEqual({ + B0AAAAAAA1: { p: 1999, r: 4.5, v: 100, pr: 1, sp: 1 }, + B0AAAAAAA2: { r: 4.5, v: 100, pr: 1, rk: 1, sp: 0 }, + }); + expect(chunk.data.expireAt).toBe(START + S.PAGE_TTL_DAYS * 86400000); + }); + + test('latest, prev and delta are replaced whole, and a failed price leaves no stale value (F-21)', () => { + const doc = w['workspaces/u1/products/B0AAAAAAA2']; + // A merge write, so sourceIds accumulates (NEW-SYNC-1); every key a map lacks is deleted. + expect(doc.merge).toBe(true); + expect(doc.data.latest.p).toBe(DELETE); + expect(doc.data.delta.p).toBe(DELETE); + expect(doc.data.delta.pPct).toBe(DELETE); + expect(doc.data.sourceIds).toEqual(['k_mug']); + const d = kept(doc.data); + expect(d.latest).toEqual({ r: 4.5, v: 100, pr: 1, rk: 1, at: Date.parse('2026-06-09T14:03:00.000Z'), runId: RUN_ID, dayKey: '2026-06-09' }); + expect(d.prev).toEqual({ p: 2100, r: 4.5, v: 95, at: Date.parse('2026-06-02T14:00:00.000Z') }); + // No price this run, so no price delta; the rest compares + expect(d.delta).toEqual({ r: 0, v: 5, days: 7 }); + }); + + test('a product seen for the first time has a null delta and no prev', () => { + const doc = kept(w['workspaces/u1/products/B0AAAAAAA1'].data); + expect(doc.delta).toBeNull(); + expect(doc.prev).toBeNull(); + expect(doc.latest.rk).toBe(3); + expect(doc.name).toBe('Mug B0AAAAAAA1'); + expect(doc).not.toHaveProperty('img'); + expect(doc).not.toHaveProperty('firstSeenAt'); + }); + + test('history writes only its own day', () => { + const h = w['workspaces/u1/products/B0AAAAAAA1/history/daily']; + expect(h.fields).toEqual(['sv', 'asin', ['d', '2026-06-09']]); + expect(h.data).toEqual({ sv: S.SV, asin: 'B0AAAAAAA1', d: { '2026-06-09': { p: 1999, r: 4.5, v: 100, pr: 1, rk: 3 } } }); + }); + + test('the run header is active with counters from the whole run', () => { + const h = w[`workspaces/u1/runs/${RUN_ID}`].data; + expect(h).toMatchObject({ + sv: S.SV, runId: RUN_ID, sourceId: 'k_mug', mk: 'US', dayKey: '2026-06-09', status: 'active', + finishedAt: null, reason: null, pagesDone: 2, maxPages: 20, pagesPlanned: null, totalResultsOnSerp: 412, + counters: { placements: 4, uniqueAsins: 3, sponsored: 1, priceParseFailures: 1, newSeen: 2 }, + }); + expect(h.source).toEqual({ type: 'keyword', sellerId: null, keyword: 'mug', url: 'https://www.amazon.com/s?k=mug' }); + }); + + test('the source never touches the fields the dashboard owns', () => { + const src = w['workspaces/u1/sources/k_mug']; + expect(src.fields.sort()).toEqual(['catalogSize', 'keyword', 'lastRunId', 'lastScrapedAt', 'sellerId', 'sourceId', 'sv', 'type', 'url']); + }); +}); + +test('a repeat on a later page is in that page\'s chunk and product writes', () => { + const writes = planEntry({ entry: entry({ kind: 'page', pageIndex: 2 }), run: run(), products, pages }); + const chunk = byPath(writes)[`workspaces/u1/runs/${RUN_ID}/pages/p0002`].data; + expect(Object.keys(chunk.items)).toEqual(['B0AAAAAAA1', 'B0AAAAAAA3']); + expect(chunk.items.B0AAAAAAA1).toMatchObject({ sp: 0, rk: 3 }); +}); + +test('firstSeenAt goes only on documents the caller found missing (F-29b)', () => { + const writes = planEntry({ entry: entry({ kind: 'page', pageIndex: 1 }), run: run(), products, pages, missing: new Set(['B0AAAAAAA1']) }); + const w = byPath(writes); + expect(w['workspaces/u1/products/B0AAAAAAA1'].data).toMatchObject({ firstSeenAt: Date.parse('2026-06-09T14:03:00.000Z'), firstRunId: RUN_ID }); + expect(w['workspaces/u1/products/B0AAAAAAA2'].data).not.toHaveProperty('firstSeenAt'); +}); + +test('first-seen candidates: new here, or last seen by nobody signed in to this account', () => { + const list = [ + product('B0NEW00001'), + product('B0MINE0001', { prev: { uid: 'u1' } }), + product('B0ANON0001', { prev: { uid: null } }), + product('B0ELSE0001', { placements: [{ page: 2, position: 1, sponsored: false, rank: 1 }] }), + ]; + expect(firstSeenCandidates(entry({ kind: 'page', pageIndex: 1 }), list)).toEqual(['B0NEW00001', 'B0ANON0001']); + expect(firstSeenCandidates(entry({ kind: 'run' }), list)).toEqual([]); +}); + +test('a run entry writes the final header and the source only', () => { + const ended = run({ state: 'blocked', reason: 'blocked', finishedAt: START + 60000 }); + const writes = planEntry({ entry: entry({ kind: 'run' }), run: ended, products, pages }); + expect(writes.map((x) => x.path.slice(2).join('/'))).toEqual([`runs/${RUN_ID}`, 'sources/k_mug']); + expect(writes[0].data).toMatchObject({ status: 'stopped', reason: 'blocked', finishedAt: START + 60000, pagesPlanned: null }); + const done = planEntry({ entry: entry({ kind: 'run' }), run: run({ state: 'done', reason: 'complete', finishedAt: START + 1 }), products, pages }); + expect(done[0].data).toMatchObject({ status: 'complete', pagesPlanned: 2 }); +}); + +test('times and array unions go through the converters the sync module passes', () => { + const writes = planEntry({ + entry: entry({ kind: 'page', pageIndex: 1 }), run: run(), products, pages, + time: (ms) => ({ toMillis: () => ms, ms }), union: (vals) => ({ union: vals }), + }); + const p = byPath(writes)['workspaces/u1/products/B0AAAAAAA1'].data; + expect(p.sourceIds).toEqual({ union: ['k_mug'] }); + expect(p.latest.at.ms).toBe(Date.parse('2026-06-09T14:03:00.000Z')); +}); + +test('a run from before 2.3 still gets its source id and day key', () => { + const old = { runId: '1749477731000-abc', source: { type: 'storefront', sellerId: 'A3K9XELT4QZ6M2', url: 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2' }, startedAt: START }; + expect(runKeys(old)).toMatchObject({ sourceId: 's_A3K9XELT4QZ6M2', source: { type: 'storefront', sellerId: 'A3K9XELT4QZ6M2' } }); + expect(runKeys(old).dayKey).toBe(S.dayKeyOf(START, new Date(START).getTimezoneOffset())); +}); + +test('with header false a page entry leaves out the run header and the source; a run entry keeps them', () => { + const page = planEntry({ entry: entry({ kind: 'page', pageIndex: 1 }), run: run(), products, pages, header: false }); + expect(page.map((x) => x.path.slice(2).join('/'))).not.toContain(`runs/${RUN_ID}`); + expect(page.map((x) => x.path.slice(2).join('/'))).not.toContain('sources/k_mug'); + const end = planEntry({ entry: entry({ kind: 'run' }), run: run({ state: 'done', reason: 'complete', finishedAt: START + 1 }), products, pages, header: false }); + expect(end.map((x) => x.path.slice(2).join('/'))).toEqual([`runs/${RUN_ID}`, 'sources/k_mug']); +}); + +describe('rules caps (EXT9-1)', () => { + test('a keyword or URL longer than the rules allow is cut to the cap', () => { + const long = 'a'.repeat(310); + const r = run({ source: { type: 'keyword', sellerId: null, keyword: long, url: `https://www.amazon.com/s?k=${long}`, sourceId: 'k_mug' } }); + const src = byPath(planEntry({ entry: entry({ kind: 'run' }), run: r, products, pages }))['workspaces/u1/sources/k_mug'].data; + expect(src.keyword.length).toBeLessThanOrEqual(RULE_CAPS.keyword); + expect(src.url.length).toBeLessThanOrEqual(RULE_CAPS.url); + }); + + test('a product the rules would refuse fails planning', () => { + const big = [product('B0AAAAAAA1', { name: 'n'.repeat(RULE_CAPS.name + 1) })]; + expect(() => planEntry({ entry: entry({ kind: 'page', pageIndex: 1 }), run: run(), products: big, pages })).toThrow(/rules cap/); + }); + + test('checkRuleCaps passes documents inside the caps', () => { + expect(() => checkRuleCaps('source', { sourceId: 'k_mug', keyword: 'mug', url: 'u', sellerId: null, lastRunId: RUN_ID })).not.toThrow(); + expect(() => checkRuleCaps('run', { reason: 'r'.repeat(101), maxPages: 5 })).toThrow(/reason/); + }); +}); diff --git a/tests/unit/sync.test.js b/tests/unit/sync.test.js index d597226..852a126 100644 --- a/tests/unit/sync.test.js +++ b/tests/unit/sync.test.js @@ -1,217 +1,294 @@ /** - * @fileoverview Offline unit test for scripts/background/sync.js syncToCloud. + * @jest-environment node * - * sync.js is authored as ESM and bundled into the service worker by esbuild. - * Here it is loaded through a tiny scoped Jest transform (see jest.config.js) - * that rewrites its import/export to CommonJS, with firebase/firestore and the - * firebase-init db fully mocked so the test never touches a real Firebase. - * - * We assert syncToCloud issues the cloud-schema writes the dashboard depends on - * (products/{asin} with latest + delta, products/{asin}/history/daily, - * runs/{runId}, sources/{sourceId}) and returns {written, runId, products}. + * scripts/background/sync.js over fake-indexeddb and an in-memory Firestore + * that applies set and mergeFields the way Firestore does. The contract test + * (tests/contract) runs the same module against the real emulator and rules. */ +jest.mock('firebase/firestore', () => ({})); -// ── Mock firebase/firestore: record every doc path and set() payload. ── -// All recording state lives INSIDE the factory (Jest hoists jest.mock above -// the file, so the factory may not close over outer variables) and is exposed -// via __writes / __committed / __reset on the mocked module. -jest.mock('firebase/firestore', () => { - const writes = []; - const state = { committed: 0 }; - const makeBatch = () => ({ - set(ref, data, opts) { - writes.push({ path: ref.__path, data, opts }); - }, - async commit() { - state.committed++; +require('fake-indexeddb/auto'); +const DB = require('../../scripts/background/db'); +const { createSync, isAuthError, isRefusal } = require('../../scripts/background/sync.js'); + +const START = Date.parse('2026-06-21T10:00:00.000Z'); +const RUN_ID = `k_wireless-mouse_${START}`; + +/** A tiny Firestore: documents in a Map, keyed by path. */ +function fakeFirestore({ failCommit = () => false } = {}) { + const docs = new Map(); + let commits = 0; + class FieldPath { constructor(...segs) { this.segs = segs; } } + const setPath = (obj, segs, value) => { + let o = obj; + segs.slice(0, -1).forEach((s) => { o[s] = o[s] && typeof o[s] === 'object' ? o[s] : {}; o = o[s]; }); + o[segs[segs.length - 1]] = value; + }; + const getPath = (obj, segs) => segs.reduce((o, s) => (o == null ? undefined : o[s]), obj); + const resolve = (v) => (v && v.__union ? v.__union : v); + const copy = (v) => (Array.isArray(v) ? v.map(copy) + : v && typeof v === 'object' ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, copy(x)])) : v); + const fs = { + doc: (_db, ...path) => ({ path: path.join('/') }), + getDoc: async (ref) => ({ exists: () => docs.has(ref.path) }), + Timestamp: { fromMillis: (ms) => ({ toMillis: () => ms, ms }) }, + arrayUnion: (...vals) => ({ __union: vals }), + deleteField: () => ({ __delete: true }), + FieldPath, + writeBatch: () => { + const ops = []; + return { + set(ref, data, opts) { ops.push({ ref, data, opts }); }, + async commit() { + const fail = failCommit(ops); + if (fail) { + const code = fail === true ? 'unavailable' : fail; + throw Object.assign(new Error(code), { code }); + } + commits++; + for (const { ref, data, opts } of ops) { + if (!opts) { docs.set(ref.path, copy(data)); continue; } + const cur = docs.get(ref.path) || {}; + if (opts.merge) { + // Deep merge, as Firestore does: maps key by key, deletes remove. + const merge = (into, from) => { + for (const [k, v] of Object.entries(from)) { + if (v && v.__delete) delete into[k]; + else if (v && v.__union) into[k] = [...new Set([...(into[k] || []), ...resolve(v)])]; + else if (v && typeof v === 'object' && !Array.isArray(v) && !v.toMillis) { + into[k] = into[k] && typeof into[k] === 'object' ? into[k] : {}; + merge(into[k], v); + } else into[k] = copy(v); + } + }; + merge(cur, data); + docs.set(ref.path, cur); + continue; + } + for (const f of opts.mergeFields) { + const segs = f instanceof FieldPath ? f.segs : [f]; + let v = getPath(data, segs); + if (v && v.__union) v = [...new Set([...(getPath(cur, segs) || []), ...resolve(v)])]; + setPath(cur, segs, copy(v)); + } + docs.set(ref.path, cur); + } + }, + }; }, - }); - return { - // doc(db, 'workspaces', uid, 'products', asin, ...) -> ref carrying its path - doc: (_db, ...segments) => ({ __path: segments.join('/') }), - writeBatch: () => makeBatch(), - serverTimestamp: () => ({ __sentinel: 'serverTimestamp' }), - arrayUnion: (...vals) => ({ __arrayUnion: vals }), - Timestamp: { fromDate: (d) => ({ __ts: d.toISOString() }) }, - // test-only handles - __writes: writes, - __state: state, - __reset: () => { - writes.length = 0; - state.committed = 0; + }; + return { fs, docs, commits: () => commits }; +} + +let dbCount = 0; +async function storeWith({ products = 3, perPage = 2, uid = 'u1', state = 'running' } = {}) { + const store = await DB.open({ name: `sync-test-${++dbCount}` }); + const pages = Math.ceil(products / perPage); + const ops = [{ + store: 'runs', + put: { + runId: RUN_ID, sourceId: 'k_wireless-mouse', dayKey: '2026-06-21', state, reason: state === 'done' ? 'complete' : null, + source: { type: 'keyword', sellerId: null, keyword: 'wireless mouse', url: 'https://www.amazon.com/s?k=wireless+mouse' }, + startedAt: START, finishedAt: state === 'done' ? START + 1000 : null, page: pages, maxPages: 20, }, + }]; + for (let n = 0; n < products; n++) { + const page = Math.floor(n / perPage) + 1; + ops.push({ + store: 'products', + put: { + runId: RUN_ID, n, pageIndex: page, asin: `B0${String(n).padStart(8, '0')}`, name: `Mouse ${n}`, + priceCents: 1000 + n, rating: 4.5, reviewCount: 10, isPrime: true, sponsored: false, organicRank: n + 1, + scrapedAt: new Date(START + n).toISOString(), placements: [{ page, position: 1, sponsored: false, rank: n + 1 }], + delta: { isNew: true, dPriceCents: null, dRating: null, dReviews: null }, prev: null, + }, + }); + } + for (let page = 1; page <= pages; page++) { + ops.push({ store: 'placements', put: { runId: RUN_ID, pageIndex: page, count: perPage, placements: perPage, kind: 'results', scrapedAt: new Date(START).toISOString(), total: products } }); + ops.push({ store: 'outbox', put: { runId: RUN_ID, kind: 'page', pageIndex: page, uid, queuedAt: START } }); + } + if (state === 'done') ops.push({ store: 'outbox', put: { runId: RUN_ID, kind: 'run', uid, queuedAt: START } }); + await store.write(ops); + return store; +} + +const quiet = { warn() {}, log() {}, error() {} }; + +test('a flush writes every queued page and empties the outbox', async () => { + const store = await storeWith({ products: 3, perPage: 2, state: 'done' }); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + const totals = await sync.flush('u1'); + // page 1: chunk + 2x2; page 2: chunk + 2x1; the header and source once, with the end + expect(totals).toEqual({ entries: 3, pages: 2, runs: 1, products: 3, writes: 5 + 3 + 2, failed: 0 }); + expect(await store.count('outbox')).toBe(0); + const run = cloud.docs.get(`workspaces/u1/runs/${RUN_ID}`); + expect(run).toMatchObject({ status: 'complete', pagesDone: 2, pagesPlanned: 2, counters: { uniqueAsins: 3 } }); + expect(cloud.docs.get('workspaces/u1/products/B000000002').sourceIds).toEqual(['k_wireless-mouse']); + expect(cloud.docs.has(`workspaces/u1/runs/${RUN_ID}/pages/p0002`)).toBe(true); + expect(cloud.docs.get('workspaces/u1/sources/k_wireless-mouse')).toMatchObject({ lastRunId: RUN_ID, catalogSize: 3 }); +}); + +test('a new product gets firstSeenAt, and a second flush never moves it (F-29b)', async () => { + const store = await storeWith({ products: 1, perPage: 1 }); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + await sync.flush('u1'); + const first = cloud.docs.get('workspaces/u1/products/B000000000'); + expect(first.firstSeenAt.ms).toBe(START); + expect(first.firstRunId).toBe(RUN_ID); + + // The same product again, as after a reinstall: new locally, known in the cloud. + cloud.docs.get('workspaces/u1/products/B000000000').firstSeenAt = { ms: 1, toMillis: () => 1 }; + await store.write([{ store: 'outbox', put: { runId: RUN_ID, kind: 'page', pageIndex: 1, uid: 'u1', queuedAt: START } }]); + await sync.flush('u1'); + expect(cloud.docs.get('workspaces/u1/products/B000000000').firstSeenAt.ms).toBe(1); +}); + +test('an entry leaves the outbox only after its own commit (F-22)', async () => { + const store = await storeWith({ products: 4, perPage: 2 }); + let calls = 0; + const cloud = fakeFirestore({ failCommit: () => ++calls === 2 }); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + await expect(sync.flush('u1')).rejects.toMatchObject({ code: 'unavailable' }); + expect((await store.getAll('outbox')).map((e) => e.pageIndex)).toEqual([2]); + await sync.flush('u1'); + expect(await store.count('outbox')).toBe(0); + expect(cloud.docs.has(`workspaces/u1/runs/${RUN_ID}/pages/p0002`)).toBe(true); +}); + +test('a page queued during a flush is written before it returns (F-22)', async () => { + const store = await storeWith({ products: 2, perPage: 2 }); + const cloud = fakeFirestore(); + let appended = false; + const onBatch = async () => { + if (appended) return; + appended = true; + await store.write([ + { store: 'products', put: { runId: RUN_ID, n: 2, pageIndex: 2, asin: 'B0LATE0001', priceCents: 5, isPrime: false, placements: [{ page: 2, position: 1, sponsored: false, rank: 3 }], delta: { isNew: true }, prev: null, scrapedAt: new Date(START).toISOString() } }, + { store: 'placements', put: { runId: RUN_ID, pageIndex: 2, count: 1, placements: 1, kind: 'last', scrapedAt: new Date(START).toISOString() } }, + { store: 'outbox', put: { runId: RUN_ID, kind: 'page', pageIndex: 2, uid: 'u1', queuedAt: START } }, + ]); }; + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, onBatch, log: quiet }); + const totals = await sync.flush('u1'); + expect(totals.pages).toBe(2); + expect(await store.count('outbox')).toBe(0); + expect(cloud.docs.get('workspaces/u1/products/B0LATE0001').latest.p).toBe(5); }); -// ── Mock the firebase-init db (sync.js imports { db } from it) ── -jest.mock('../../scripts/background/firebase-init.js', () => ({ db: { __db: true } }), { - virtual: true, +test('a second flush while one runs waits for it and does not write twice', async () => { + const store = await storeWith({ products: 2, perPage: 1 }); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + const [a, b] = await Promise.all([sync.flush('u1'), sync.flush('u1')]); + expect(a).toBe(b); + expect(a.pages).toBe(2); + expect(cloud.commits()).toBe(2); }); -const firestoreMock = require('firebase/firestore'); -const writes = firestoreMock.__writes; -const { syncToCloud } = require('../../scripts/background/sync.js'); +test('entries of another account are left alone (F-29e)', async () => { + const store = await storeWith({ products: 2, perPage: 2, uid: 'someone-else' }); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + expect(await sync.pending('u1')).toBe(0); + expect(await sync.flush('u1')).toMatchObject({ entries: 0, writes: 0 }); + expect(cloud.docs.size).toBe(0); + expect(await sync.pending('someone-else')).toBe(1); + expect(await sync.flush(null)).toMatchObject({ entries: 0 }); +}); -const UID = 'user_abc'; +test('big pages are split into batches under the Firestore limit', async () => { + const store = await storeWith({ products: 60, perPage: 60 }); + const cloud = fakeFirestore(); + const sizes = []; + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, batchLimit: 50, onBatch: async ({ writes }) => sizes.push(writes), log: quiet }); + const totals = await sync.flush('u1'); + expect(totals.writes).toBe(1 + 120 + 2); + expect(sizes).toEqual([50, 50, 23]); +}); -function seedBundle() { - return { - scrapeRunMeta: { - type: 'keyword', - keyword: 'wireless mouse', - sellerId: null, - url: 'https://www.amazon.com/s?k=wireless+mouse', - startedAt: '2026-06-21T10:00:00.000Z', - }, - scrapeRunPages: [{ pageIndex: 0 }, { pageIndex: 1 }], - syncQueue: [ - { - // first-sight product - asin: 'B0NEW00001', - name: 'New Mouse', - url: 'https://www.amazon.com/dp/B0NEW00001', - priceCents: 1999, - rating: 4.5, - reviewCount: 120, - isPrime: true, - scrapedAt: '2026-06-21T10:00:05.000Z', - delta: { isNew: true, dPriceCents: null, dRating: null, dReviews: null }, - }, - { - // returning product with a price drop - asin: 'B0SEEN00002', - name: 'Seen Mouse', - url: 'https://www.amazon.com/dp/B0SEEN00002', - priceCents: 1799, - rating: 4.2, - reviewCount: 300, - isPrime: false, - scrapedAt: '2026-06-21T10:00:06.000Z', - delta: { isNew: false, dPriceCents: -200, dRating: 0, dReviews: 20 }, - }, - ], - }; -} +test('an entry whose run is gone is dropped', async () => { + const store = await DB.open({ name: `sync-test-${++dbCount}` }); + await store.write([{ store: 'outbox', put: { runId: 'gone', kind: 'page', pageIndex: 1, uid: 'u1' } }]); + const sync = createSync({ db: {}, openStore: async () => store, fs: fakeFirestore().fs, log: quiet }); + expect(await sync.flush('u1')).toMatchObject({ entries: 1, pages: 0 }); + expect(await store.count('outbox')).toBe(0); +}); + +test('an entry that fails validation is dropped instead of blocking the queue', async () => { + const store = await DB.open({ name: `sync-test-${++dbCount}` }); + // A run from before 2.3: its id does not start with its source id. + await store.write([ + { store: 'runs', put: { runId: '1717000000000-abc1234', state: 'updated', reason: 'updated', startedAt: 1717000000000, finishedAt: 1717000100000, maxPages: 5, page: 1, source: { type: 'keyword', keyword: 'lamp', sourceId: 'k_lamp' } } }, + { store: 'outbox', put: { runId: '1717000000000-abc1234', kind: 'run', uid: 'u1' } } + ]); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + expect(await sync.flush('u1')).toMatchObject({ entries: 1, writes: 0 }); + expect(await store.count('outbox')).toBe(0); +}); + +test('auth errors are told apart from network and rules errors', () => { + expect(isAuthError({ code: 'permission-denied' })).toBe(false); + expect(isAuthError({ code: 'unauthenticated' })).toBe(true); + expect(isAuthError({ code: 'auth/user-token-expired' })).toBe(true); + expect(isAuthError({ code: 'unavailable' })).toBe(false); + expect(isAuthError(null)).toBe(false); +}); + +test('a product seen by two sources lists both (NEW-SYNC-1)', async () => { + const store = await storeWith({ products: 1, perPage: 1, state: 'done' }); + const cloud = fakeFirestore(); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + await sync.flush('u1'); + // The same ASIN in a storefront run. + const RUN2 = `s_A3K9XELT4QZ6M2_${START + 5000}`; + await store.write([ + { store: 'runs', put: { runId: RUN2, sourceId: 's_A3K9XELT4QZ6M2', dayKey: '2026-06-21', state: 'done', reason: 'complete', source: { type: 'storefront', sellerId: 'A3K9XELT4QZ6M2', keyword: null, url: 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2' }, startedAt: START + 5000, finishedAt: START + 6000, page: 1, maxPages: 20 } }, + { store: 'products', put: { runId: RUN2, n: 0, pageIndex: 1, asin: 'B000000000', name: 'Mouse 0', priceCents: 999, rating: 4.5, reviewCount: 10, isPrime: true, sponsored: false, organicRank: 1, scrapedAt: new Date(START + 5000).toISOString(), placements: [{ page: 1, position: 1, sponsored: false, rank: 1 }], delta: { isNew: false }, prev: null } }, + { store: 'placements', put: { runId: RUN2, pageIndex: 1, count: 1, placements: 1, kind: 'last', scrapedAt: new Date(START + 5000).toISOString() } }, + { store: 'outbox', put: { runId: RUN2, kind: 'page', pageIndex: 1, uid: 'u1', queuedAt: START } }, + { store: 'outbox', put: { runId: RUN2, kind: 'run', uid: 'u1', queuedAt: START } }, + ]); + await sync.flush('u1'); + expect(cloud.docs.get('workspaces/u1/products/B000000000').sourceIds.sort()).toEqual(['k_wireless-mouse', 's_A3K9XELT4QZ6M2']); + expect(cloud.docs.get('workspaces/u1/products/B000000000').latest.p).toBe(999); +}); + +test('an entry the rules refuse is marked failed and the entries behind it still sync (EXT9-1)', async () => { + const store = await storeWith({ products: 4, perPage: 2, state: 'done' }); + // The rules refuse page 1 of the run, every time. + const cloud = fakeFirestore({ failCommit: (ops) => ops.some((o) => o.ref.path.endsWith('/pages/p0001')) && 'permission-denied' }); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + const totals = await sync.flush('u1'); + expect(totals).toMatchObject({ entries: 2, pages: 1, failed: 1 }); + expect(cloud.docs.has(`workspaces/u1/runs/${RUN_ID}/pages/p0002`)).toBe(true); + expect(cloud.docs.get(`workspaces/u1/runs/${RUN_ID}`)).toMatchObject({ status: 'complete' }); + const left = await store.getAll('outbox'); + expect(left).toHaveLength(1); + expect(left[0]).toMatchObject({ pageIndex: 1, failed: { code: 'permission-denied' } }); + expect(await sync.pending('u1')).toBe(0); + expect(await sync.failed('u1')).toBe(1); + + // An automatic flush leaves it alone; Export tries it again. + expect(await sync.flush('u1')).toMatchObject({ entries: 0, failed: 0 }); + expect(await sync.flush('u1', { retryFailed: true })).toMatchObject({ entries: 0, failed: 1 }); +}); + +test('when every entry is refused nothing is marked and the error goes to the caller', async () => { + const store = await storeWith({ products: 2, perPage: 1 }); + const cloud = fakeFirestore({ failCommit: () => 'permission-denied' }); + const sync = createSync({ db: {}, openStore: async () => store, fs: cloud.fs, log: quiet }); + await expect(sync.flush('u1')).rejects.toMatchObject({ code: 'permission-denied' }); + expect(await sync.pending('u1')).toBe(2); + expect(await sync.failed('u1')).toBe(0); +}); -const findWrite = (path) => writes.find((w) => w.path === path); - -describe('sync.js — syncToCloud', () => { - beforeEach(() => { - firestoreMock.__reset(); - }); - - test('returns {written:0, runId:null, products:0} for an empty queue', async () => { - const res = await syncToCloud(UID, { syncQueue: [] }); - expect(res).toEqual({ written: 0, runId: null, products: 0 }); - expect(writes).toHaveLength(0); - }); - - test('derives a canonical keyword runId/sourceId from run meta', async () => { - const res = await syncToCloud(UID, seedBundle()); - const sourceId = 'k_wireless-mouse'; - const startMs = Date.parse('2026-06-21T10:00:00.000Z'); - expect(res.runId).toBe(`${sourceId}_${startMs}`); - }); - - test('writes products/{asin} with latest (incl. dayKey) + sourceIds', async () => { - await syncToCloud(UID, seedBundle()); - const p = findWrite(`workspaces/${UID}/products/B0NEW00001`); - expect(p).toBeDefined(); - expect(p.opts).toEqual({ merge: true }); - expect(p.data.asin).toBe('B0NEW00001'); - expect(p.data.mk).toBe('US'); - expect(p.data.latest.p).toBe(1999); - expect(p.data.latest.pr).toBe(1); // isPrime -> 1 - expect(p.data.latest.dayKey).toBe('2026-06-21'); - expect(p.data.sourceIds).toEqual({ __arrayUnion: ['k_wireless-mouse'] }); - // first-sight stamps present on a new product - expect(p.data.firstRunId).toBeDefined(); - expect(p.data.firstSeenAt).toBeDefined(); - }); - - test('writes a delta block for a returning product, not for a new one', async () => { - await syncToCloud(UID, seedBundle()); - const seen = findWrite(`workspaces/${UID}/products/B0SEEN00002`); - const fresh = findWrite(`workspaces/${UID}/products/B0NEW00001`); - expect(seen.data.delta).toMatchObject({ p: -200, v: 20 }); - expect(seen.data.delta.pPct).toBeCloseTo(-10, 1); // -200 on prior 1999 - expect(fresh.data.delta).toBeUndefined(); - expect(fresh.data.firstRunId).toBeDefined(); - // returning product gets no first-sight stamp - expect(seen.data.firstRunId).toBeUndefined(); - }); - - test('writes the date-keyed history/daily point per product', async () => { - await syncToCloud(UID, seedBundle()); - const h = findWrite(`workspaces/${UID}/products/B0NEW00001/history/daily`); - expect(h).toBeDefined(); - expect(h.opts).toEqual({ merge: true }); - expect(h.data.d['2026-06-21']).toMatchObject({ p: 1999, r: 4.5, v: 120, pr: 1 }); - }); - - test('writes the run header doc with counters', async () => { - await syncToCloud(UID, seedBundle()); - const startMs = Date.parse('2026-06-21T10:00:00.000Z'); - const runId = `k_wireless-mouse_${startMs}`; - const run = findWrite(`workspaces/${UID}/runs/${runId}`); - expect(run).toBeDefined(); - expect(run.data.runId).toBe(runId); - expect(run.data.sourceId).toBe('k_wireless-mouse'); - expect(run.data.status).toBe('complete'); - expect(run.data.dayKey).toBe('2026-06-21'); - expect(run.data.counters.placements).toBe(2); - expect(run.data.counters.uniqueAsins).toBe(2); - expect(run.data.counters.newSeen).toBe(1); - expect(run.data.pagesDone).toBe(2); - }); - - test('counts every placement and sponsored card, and skips a missing name (F-27, F-28)', async () => { - const bundle = seedBundle(); - bundle.syncQueue[0].placements = [ - { page: 1, position: 1, sponsored: true, rank: null }, - { page: 1, position: 4, sponsored: false, rank: 3 }, - ]; - bundle.syncQueue[1].name = null; - bundle.syncQueue[1].url = 'https://www.amazon.com/sspa/click?url=x'; - await syncToCloud(UID, bundle); - const startMs = Date.parse('2026-06-21T10:00:00.000Z'); - const run = findWrite(`workspaces/${UID}/runs/k_wireless-mouse_${startMs}`); - expect(run.data.counters).toMatchObject({ placements: 3, uniqueAsins: 2, sponsored: 1 }); - const seen = findWrite(`workspaces/${UID}/products/B0SEEN00002`); - expect(seen.data).not.toHaveProperty('name'); - expect(seen.data.url).toBe('https://www.amazon.com/dp/B0SEEN00002'); - }); - - test('writes the source spine doc with lastRunId', async () => { - await syncToCloud(UID, seedBundle()); - const src = findWrite(`workspaces/${UID}/sources/k_wireless-mouse`); - expect(src).toBeDefined(); - expect(src.data.sourceId).toBe('k_wireless-mouse'); - expect(src.data.type).toBe('keyword'); - expect(src.data.keyword).toBe('wireless mouse'); - expect(src.data.lastRunId).toMatch(/^k_wireless-mouse_/); - }); - - test('returns the expected write tally (2 per product + 2 headers)', async () => { - const res = await syncToCloud(UID, seedBundle()); - expect(res.products).toBe(2); - expect(res.written).toBe(2 * 2 + 2); // products*2 + run + source - // every batch was committed (1 product batch + 1 header batch) - expect(firestoreMock.__state.committed).toBe(2); - }); - - test('derives a storefront sourceId (s_{sellerId}) from seller meta', async () => { - const bundle = seedBundle(); - bundle.scrapeRunMeta = { - type: 'storefront', - sellerId: 'A123XYZ', - keyword: null, - startedAt: '2026-06-21T10:00:00.000Z', - }; - await syncToCloud(UID, bundle); - const src = findWrite(`workspaces/${UID}/sources/s_A123XYZ`); - expect(src).toBeDefined(); - expect(src.data.type).toBe('storefront'); - expect(src.data.sellerId).toBe('A123XYZ'); - }); +test('refusals are told apart from errors worth retrying', () => { + expect(isRefusal({ code: 'permission-denied' })).toBe(true); + expect(isRefusal({ code: 'invalid-argument' })).toBe(true); + expect(isRefusal({ code: 'unavailable' })).toBe(false); + expect(isRefusal({ code: 'resource-exhausted' })).toBe(false); }); diff --git a/tools/contract.mjs b/tools/contract.mjs new file mode 100644 index 0000000..6d8aed0 --- /dev/null +++ b/tools/contract.mjs @@ -0,0 +1,167 @@ +// tools/contract.mjs: runs the sync contract test against the Firebase +// emulators, with the dashboard repo's firestore.rules. +// +// node tools/contract.mjs +// +// The rules come from PROSCAN_RULES, else ../web/firestore.rules or +// ../proscan-web/firestore.rules next to this repo. Ports default away from +// the dashboard's (EMU_AUTH_PORT 9299, EMU_FIRESTORE_PORT 8288) so both can +// be checked out side by side. Project is demo-proscan: emulators only. +// +// It refuses to start when any of its ports is already taken, and on exit it +// only kills processes that listen on its ports and started after it did, +// so another emulator, dev server or agent on those ports is never touched. + +import { spawn, execSync } from 'node:child_process'; +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const TEST = path.join(ROOT, 'tests', 'contract', 'sync.contract.mjs'); +const TIMEOUT_MS = 7 * 60 * 1000; + +const port = (name, fallback) => { + const n = Number(process.env[name] || fallback); + if (!Number.isInteger(n) || n <= 0) throw new Error(`${name} is not a port`); + return n; +}; +const ports = { + auth: port('EMU_AUTH_PORT', 9299), + firestore: port('EMU_FIRESTORE_PORT', 8288), + websocket: port('EMU_WEBSOCKET_PORT', 9250), + hub: port('EMU_HUB_PORT', 4488), + logging: port('EMU_LOGGING_PORT', 4588), +}; + +export function findRules(env = process.env) { + const candidates = env.PROSCAN_RULES + ? [path.resolve(env.PROSCAN_RULES)] + : [path.join(ROOT, '..', 'web', 'firestore.rules'), path.join(ROOT, '..', 'proscan-web', 'firestore.rules')]; + const found = candidates.find((p) => fs.existsSync(p)); + if (!found) throw new Error(`firestore.rules not found; tried ${candidates.join(', ')}. Set PROSCAN_RULES.`); + return found; +} + +/** Resolves true when nothing can be bound to 127.0.0.1:`port`. */ +function portTaken(port) { + return new Promise((resolve) => { + const srv = net.createServer(); + srv.once('error', () => resolve(true)); + srv.listen(port, '127.0.0.1', () => srv.close(() => resolve(false))); + }); +} + +/** The ports of `list` something already listens on. */ +export async function busyPorts(list) { + const taken = []; + for (const p of list) if (await portTaken(p)) taken.push(p); + return taken; +} + +/** {port: Set(pid)} for every LISTENING socket on `list`, on any address (Windows only). */ +function listeners(list) { + const out = new Map(); + if (process.platform !== 'win32') return out; + let text = ''; + try { + text = execSync('netstat -ano -p tcp', { encoding: 'utf8' }); + } catch { + return out; + } + const wanted = new Set(list.map(String)); + for (const line of text.split(/\r?\n/)) { + const m = line.trim().match(/^TCP\s+\S+:(\d+)\s+\S+\s+LISTENING\s+(\d+)/); + if (m && wanted.has(m[1])) { + if (!out.has(m[1])) out.set(m[1], new Set()); + out.get(m[1]).add(m[2]); + } + } + return out; +} + +/** When process `pid` started, in ms, or null when that cannot be read. */ +function startedAt(pid) { + try { + const iso = execSync( + `powershell -NoProfile -Command "(Get-Process -Id ${Number(pid)}).StartTime.ToUniversalTime().ToString('o')"`, + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + const t = Date.parse(iso); + return Number.isFinite(t) ? t : null; + } catch { + return null; + } +} + +/** + * On Windows java outlives firebase, so the emulators this run started can + * keep our ports. Kills only listeners on our ports that started after + * `since`; anything older belongs to someone else. + */ +function freeOurPorts(since) { + for (const pids of listeners(Object.values(ports)).values()) { + for (const pid of pids) { + const t = startedAt(pid); + if (t === null || t < since) continue; + try { + execSync(`taskkill /PID ${pid} /T /F`, { stdio: 'ignore' }); + } catch { + /* already gone */ + } + } + } +} + +async function main() { + const rules = findRules(); + const taken = await busyPorts(Object.values(ports)); + if (taken.length) { + console.error(`[contract] FAIL: port${taken.length > 1 ? 's' : ''} ${taken.join(', ')} already in use. Nothing was started or stopped.`); + console.error('[contract] Pick free ports with EMU_AUTH_PORT, EMU_FIRESTORE_PORT, EMU_WEBSOCKET_PORT, EMU_HUB_PORT and EMU_LOGGING_PORT.'); + process.exit(2); + } + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'proscan-contract-')); + fs.copyFileSync(rules, path.join(dir, 'firestore.rules')); + fs.writeFileSync(path.join(dir, 'firebase.json'), JSON.stringify({ + firestore: { rules: 'firestore.rules' }, + emulators: { + auth: { host: '127.0.0.1', port: ports.auth }, + firestore: { host: '127.0.0.1', port: ports.firestore, websocketPort: ports.websocket }, + hub: { host: '127.0.0.1', port: ports.hub }, + logging: { host: '127.0.0.1', port: ports.logging }, + ui: { enabled: false }, + singleProjectMode: true, + }, + }, null, 2)); + console.log(`[contract] rules ${rules}`); + console.log(`[contract] auth ${ports.auth}, firestore ${ports.firestore}`); + + const cmd = `node --test --test-concurrency=1 "${TEST}"`; + const spawnedAt = Date.now() - 1000; + const child = spawn('firebase', [ + 'emulators:exec', '--only', 'auth,firestore', '--project', 'demo-proscan', + '--config', path.join(dir, 'firebase.json'), JSON.stringify(cmd), + ], { cwd: dir, stdio: 'inherit', shell: true, env: { ...process.env, PROSCAN_CONTRACT_RULES: rules } }); + + const timer = setTimeout(() => { + console.error('[contract] timed out'); + child.kill('SIGTERM'); + }, TIMEOUT_MS); + + const cleanup = () => { + clearTimeout(timer); + freeOurPorts(spawnedAt); + fs.rmSync(dir, { recursive: true, force: true }); + }; + child.on('exit', (code, signal) => { + cleanup(); + process.exit(signal ? 1 : (code ?? 1)); + }); + for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig)); +} + +const invokedDirectly = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) main().catch((err) => { console.error(`[contract] FAIL: ${err.message}`); process.exit(1); }); diff --git a/tools/tests/contract.test.mjs b/tools/tests/contract.test.mjs new file mode 100644 index 0000000..bf43945 --- /dev/null +++ b/tools/tests/contract.test.mjs @@ -0,0 +1,16 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import net from 'node:net'; +import { busyPorts } from '../contract.mjs'; + +test('busyPorts names a port someone else listens on, and nothing else', async () => { + const srv = net.createServer(); + await new Promise((r) => srv.listen(0, '127.0.0.1', r)); + const { port } = srv.address(); + try { + assert.deepEqual(await busyPorts([port]), [port]); + } finally { + await new Promise((r) => srv.close(r)); + } + assert.deepEqual(await busyPorts([port]), []); +});