diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 974f3ac..ca0098a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,21 @@ permissions: jobs: check: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 25 + env: + PROSCAN_EXT: ${{ github.workspace }}/extension steps: - uses: actions/checkout@v4 + # The schema check, the sync rules test and the e2e run the + # extension's own schema, engine and sync module. Its main branch has + # to carry packages/schema (extension 2.3). + - uses: actions/checkout@v4 + with: + repository: EnesYilmazcode/AmazonSellerScraper + ref: main + path: extension + - uses: actions/setup-node@v4 with: node-version: 22 @@ -33,11 +44,19 @@ jobs: key: firebase-emulators-15.8.0 - run: npm ci + - run: npm ci + working-directory: extension - run: npm install -g firebase-tools@15.8.0 - name: Typecheck run: npm run typecheck + - name: Schema matches the extension + run: npm run check:schema + + - name: Unit tests + run: npm run test:unit + - name: Build run: npm run build @@ -54,3 +73,9 @@ jobs: env: VITE_USE_EMULATOR: 'true' run: npm run build && node scripts/check-bundle.mjs --emulator + + - name: Install Chromium for the e2e + run: npx playwright install --with-deps chromium + + - name: End to end (emulator, production bundle) + run: npm run test:e2e diff --git a/.gitignore b/.gitignore index e3dddfb..941dc02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ firebase-debug.*.log* # Build output (generated by `npm run build`) dist/ +# Emulator bundle built by `npm run test:e2e` +dist-e2e/ # Firebase cache .firebase/ diff --git a/README.md b/README.md index b780c07..5eabd3c 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,11 @@ npm install # one-time npm run dev # landing page dev server npm run dev:dashboard # dashboard dev server, talks to the emulators npm run emulators # auth + firestore + hosting emulators (project demo-proscan) -npm run seed # demo data into the running emulators +npm run seed # scans written by the extension's own sync module +npm run seed:fixture # the older hand-made fixture (predates the schema) npm run build # clean dist/, then build landing + dashboard -npm test # typecheck, build, bundle check, rules tests +npm test # typecheck, schema check, unit tests, build, bundle check, rules tests +npm run test:e2e # extension sync seeds the emulator, the production bundle is checked npm run test:hosting # rewrites and cache headers (Linux or WSL only) npm run deploy # hosting + firestore rules + indexes to proscanbot ``` @@ -61,6 +63,31 @@ refuses a dirty tree or a branch other than `main`, so the stamp always names a real commit. CI (`.github/workflows/ci.yml`) runs the same checks on every pull request. +## The shared schema + +`packages/schema/index.js` is the extension's cloud schema, copied byte for +byte from `packages/schema/index.js` in the extension repo. It holds the +document shapes, validators and id builders for everything the extension +writes. `index.d.ts` next to it is ours. `npm run check:schema` fails when +the two copies differ; to update, copy the extension's file over ours and +adjust `index.d.ts`. + +The dashboard reads every document through a converter that runs the +schema's validator (`dashboard/src/lib/checked.ts`). A document that fails +is left out of the view and counted in a notice. `firestore.rules` checks +the same shapes on write. + +Several checks run the extension's real code from its checkout: the schema +check, the sync rules test, the e2e and `npm run seed`. They look for it +in `PROSCAN_EXT`, else `../ext` or `../AmazonSellerScraper`. Set +`PROSCAN_ALLOW_NO_EXT=1` to skip them where there is no checkout. CI checks +out the extension's `main`. + +`npm run test:e2e` builds the dashboard with `VITE_USE_EMULATOR=true` into +`dist-e2e/`, serves it with `vite preview` and drives it with Playwright. +`QA_CHROMIUM` points at an installed Chromium when Playwright's own is +missing. Screenshots and the exported workbook land in `.screenshots/e2e/`. + ## Revamp Adds sign-in, saving client/competitor data, and integration with the ProScan diff --git a/dashboard/src/auth/ExtensionPassword.tsx b/dashboard/src/auth/ExtensionPassword.tsx new file mode 100644 index 0000000..44ea6a8 --- /dev/null +++ b/dashboard/src/auth/ExtensionPassword.tsx @@ -0,0 +1,232 @@ +// "Set a password for the extension" (F-50). The extension signs in with +// email and password only, so an account made with Google has nothing to +// type there and its workspace never fills. Linking an email credential to +// the same account gives it a password without making a second account. + +import { useState, type FormEvent } from 'react'; +import { + EmailAuthProvider, + linkWithCredential, + reauthenticateWithPopup, + updatePassword, + type User, +} from 'firebase/auth'; +import { FirebaseError } from 'firebase/app'; +import { googleProvider } from '../firebase'; +import { useAuthUser } from '../lib/hooks'; +import Button from '../components/Button'; +import './extpw.css'; + +export const MIN_PASSWORD = 8; + +export function hasPassword(user: User): boolean { + return user.providerData.some((p) => p.providerId === 'password'); +} + +function message(err: unknown): string { + const code = codeOf(err); + switch (code) { + case 'auth/weak-password': + return `Use at least ${MIN_PASSWORD} characters.`; + case 'auth/credential-already-in-use': + case 'auth/email-already-in-use': + return 'Another ProScan account already uses this email with a password. Sign in to the extension with that password, or reset it from the extension.'; + case 'auth/popup-blocked': + return 'Your browser blocked the Google window. Allow pop-ups and try again.'; + case 'auth/popup-closed-by-user': + case 'auth/cancelled-popup-request': + return 'Google sign-in was closed before it finished. Try again.'; + case 'auth/network-request-failed': + return 'Network error. Check your connection and try again.'; + default: + return code ? `Couldn't set the password (${code}).` : "Couldn't set the password."; + } +} + +const codeOf = (err: unknown) => (err instanceof FirebaseError ? err.code : ''); + +async function linkOnce(user: User, password: string): Promise { + try { + await linkWithCredential(user, EmailAuthProvider.credential(user.email!, password)); + } catch (err) { + if (codeOf(err) === 'auth/provider-already-linked') return; + // The account's own email counts as taken where the backend checks + // every account, as the Auth emulator does. With one account per email + // the only holder is this account, and a password set on it adds the + // same email provider. + if (codeOf(err) !== 'auth/email-already-in-use') throw err; + await updatePassword(user, password); + } +} + +async function link(user: User, password: string): Promise { + try { + await linkOnce(user, password); + } catch (err) { + if (codeOf(err) !== 'auth/requires-recent-login') throw err; + // An old Google session: confirm it, then try again. + await reauthenticateWithPopup(user, googleProvider); + await linkOnce(user, password); + } + await user.reload(); +} + +const DISMISS_KEY = (uid: string) => `proscan:extension-password-later:${uid}`; + +function readLater(uid: string): boolean { + try { + return window.localStorage.getItem(DISMISS_KEY(uid)) === '1'; + } catch { + return false; + } +} + +function writeLater(uid: string, later: boolean): void { + try { + if (later) window.localStorage.setItem(DISMISS_KEY(uid), '1'); + else window.localStorage.removeItem(DISMISS_KEY(uid)); + } catch { + /* storage blocked: the card just comes back next visit */ + } +} + +/** Shown above every view to a signed-in user who has no password yet. */ +export default function ExtensionPassword() { + const { user } = useAuthUser(); + const [open, setOpen] = useState(false); + const [later, setLater] = useState(() => (user ? readLater(user.uid) : false)); + const [password, setPassword] = useState(''); + const [confirm, setConfirm] = useState(''); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const [done, setDone] = useState(false); + + if (!user || !user.email) return null; + if (done) { + return ( +
+ Password set. In the extension, sign in with {user.email} and + this password. Your scans will sync here. + +
+ ); + } + if (hasPassword(user)) return null; + + if (later && !open) { + return ( +
+ The extension needs a password for {user.email}. + +
+ ); + } + + const submit = async (e: FormEvent) => { + e.preventDefault(); + if (busy) return; + setError(null); + if (password.length < MIN_PASSWORD) { + setError(`Use at least ${MIN_PASSWORD} characters.`); + return; + } + if (password !== confirm) { + setError("The two passwords don't match."); + return; + } + setBusy(true); + try { + await link(user, password); + setPassword(''); + setConfirm(''); + setOpen(false); + setDone(true); + } catch (err) { + console.error('[proscan] linking a password failed', err); + setError(message(err)); + } finally { + setBusy(false); + } + }; + + return ( +
+
+

+ Set a password for the extension +

+

+ The ProScan extension signs in with an email and password. You signed in here with + Google, so the extension has nothing to sign in with yet and no scans can reach this + dashboard. Set a password for {user.email}, then use it in + the extension. It stays the same account. +

+
+ {open ? ( +
+ + setPassword(e.target.value)} + placeholder={`At least ${MIN_PASSWORD} characters`} + /> + + setConfirm(e.target.value)} + /> + {error ? ( +
+ {error} +
+ ) : null} +
+ + +
+
+ ) : ( +
+ + +
+ )} +
+ ); +} diff --git a/dashboard/src/auth/SignIn.tsx b/dashboard/src/auth/SignIn.tsx index d161943..d24592d 100644 --- a/dashboard/src/auth/SignIn.tsx +++ b/dashboard/src/auth/SignIn.tsx @@ -10,6 +10,7 @@ import { auth, googleProvider, USE_EMULATOR } from '../firebase'; import Button from '../components/Button'; import { RadarIcon } from '../components/EmptyState'; import GoogleButton from './GoogleButton'; +import { MIN_PASSWORD } from './ExtensionPassword'; import ScanBoard from './ScanBoard'; import './auth.css'; @@ -46,7 +47,7 @@ function friendlyAuthError(err: unknown): string { case 'auth/email-already-in-use': return 'An account with this email already exists — sign in instead.'; case 'auth/weak-password': - return 'Password must be at least 6 characters.'; + return `Password must be at least ${MIN_PASSWORD} characters.`; case 'auth/too-many-requests': return 'Too many attempts. Wait a minute and try again.'; case 'auth/network-request-failed': @@ -141,8 +142,12 @@ export default function SignIn() { const submit = async (e: FormEvent) => { e.preventDefault(); if (busy || googleBusy) return; - setBusy(true); clearMessages(); + if (mode === 'create' && password.length < MIN_PASSWORD) { + setError(`Password must be at least ${MIN_PASSWORD} characters.`); + return; + } + setBusy(true); try { if (mode === 'signin') { await signInWithEmailAndPassword(auth, email.trim(), password); @@ -260,7 +265,7 @@ export default function SignIn() { className="auth__input" type={showPw ? 'text' : 'password'} autoComplete={mode === 'signin' ? 'current-password' : 'new-password'} - placeholder={mode === 'create' ? 'At least 6 characters' : 'Enter your password'} + placeholder={mode === 'create' ? `At least ${MIN_PASSWORD} characters` : 'Enter your password'} value={password} onChange={(e) => setPassword(e.target.value)} required diff --git a/dashboard/src/auth/extpw.css b/dashboard/src/auth/extpw.css new file mode 100644 index 0000000..3c996ad --- /dev/null +++ b/dashboard/src/auth/extpw.css @@ -0,0 +1,81 @@ +/* "Set a password for the extension" card, above the routed view. */ +.extpw { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: var(--space-4); + margin: 0 0 var(--space-4); + padding: var(--space-4); + background: var(--surface); + border: 1px solid var(--gold); + border-radius: var(--radius-md); + box-shadow: var(--shadow); + color: var(--ink); +} +.extpw--slim, +.extpw--done { + align-items: center; + gap: var(--space-2); + padding: var(--space-2) var(--space-3); + font-size: 13px; + border-color: var(--line); +} +.extpw--done { + border-color: var(--green); +} +.extpw__text { + flex: 1 1 320px; +} +.extpw__title { + margin: 0 0 var(--space-1); + font-size: 15px; + font-weight: 700; +} +.extpw__body { + margin: 0; + color: var(--muted); + font-size: 13px; + line-height: 1.5; +} +.extpw__form { + flex: 1 1 260px; + display: flex; + flex-direction: column; + gap: var(--space-1); + max-width: 320px; +} +.extpw__label { + font-size: 12px; + color: var(--muted); +} +.extpw__input { + height: 34px; + padding: 0 var(--space-2); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + font: inherit; + color: var(--ink); + background: var(--surface); +} +.extpw__input:focus { + outline: 2px solid var(--gold-glow); + border-color: var(--gold); +} +.extpw__error { + color: var(--red); + font-size: 13px; +} +.extpw__actions { + display: flex; + gap: var(--space-2); + margin-top: var(--space-2); +} +.extpw__link { + border: 0; + padding: 0; + background: none; + color: var(--ink); + font: inherit; + text-decoration: underline; + cursor: pointer; +} diff --git a/dashboard/src/components/SchemaNotice.tsx b/dashboard/src/components/SchemaNotice.tsx new file mode 100644 index 0000000..3f7e3da --- /dev/null +++ b/dashboard/src/components/SchemaNotice.tsx @@ -0,0 +1,27 @@ +import type { SchemaProblem } from '../lib/checked'; +import './components.css'; + +/** Says how many documents were left out for failing the schema check, + * with the first one's path and problems. Renders nothing when none were. */ +export default function SchemaNotice({ invalid }: { invalid: SchemaProblem[] }) { + if (invalid.length === 0) return null; + const first = invalid[0]; + const noun = invalid.length === 1 ? 'document' : 'documents'; + return ( +
+ + {invalid.length.toLocaleString('en-US')} {noun} left out + {' '} + because {invalid.length === 1 ? 'it does' : 'they do'} not match the ProScan schema. An + older extension build may have written {invalid.length === 1 ? 'it' : 'them'}. +
+ {first.path} +
    + {first.problems.slice(0, 6).map((p) => ( +
  • {p}
  • + ))} +
+
+
+ ); +} diff --git a/dashboard/src/components/components.css b/dashboard/src/components/components.css index 91c2cf0..1f37c42 100644 --- a/dashboard/src/components/components.css +++ b/dashboard/src/components/components.css @@ -446,3 +446,28 @@ line-height: 1; cursor: pointer; } + +/* ── notice: a thin banner above a view ─────────────────────────── */ +.notice { + margin: 0 0 var(--space-3); + padding: var(--space-2) var(--space-3); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + background: var(--surface); + color: var(--ink); + font-size: 13px; + line-height: 1.45; +} +.notice--warn { + border-color: var(--gold); + background: var(--gold-wash); +} +.notice__details { + margin-top: var(--space-1); + color: var(--muted); +} +.notice__details ul { + margin: var(--space-1) 0 0; + padding-left: var(--space-4); + font-size: 12px; +} diff --git a/dashboard/src/features/board/BoardToolbar.tsx b/dashboard/src/features/board/BoardToolbar.tsx index 68acb01..530f467 100644 --- a/dashboard/src/features/board/BoardToolbar.tsx +++ b/dashboard/src/features/board/BoardToolbar.tsx @@ -1,7 +1,7 @@ -// Delta-board toolbar — search-in-loaded-set, Latest|Movers segmented -// toggle (gold active), source scope dropdown (synced to ?source=), the -// buyer-semantics legend popover and the XLSX export of the currently -// visible sorted rows. Pure presentation: all state lives in the route. +// Delta-board toolbar — search, Latest|Movers segmented toggle (gold +// active), source scope dropdown (synced to ?source=), the buyer-semantics +// legend popover and the XLSX export of the whole scope. Pure +// presentation: all state lives in the route. import { useEffect, useRef, useState } from 'react'; import { clsx } from 'clsx'; @@ -83,8 +83,10 @@ export interface BoardToolbarProps { sourceId: string | null; sources: Source[]; onSourceChange: (sourceId: string | null) => void; - /** Currently visible rows in sorted order — exactly what Export writes. */ - exportRows: Product[]; + /** Fetches every row Export writes: the whole scope, searched and sorted. */ + exportLoad: (onProgress: (loaded: number) => void) => Promise; + /** Rows Export will write, when known before fetching. */ + exportCount: number | null; } export default function BoardToolbar({ @@ -95,7 +97,8 @@ export default function BoardToolbar({ sourceId, sources, onSourceChange, - exportRows, + exportLoad, + exportCount, }: BoardToolbarProps) { const options = [...sources].sort((a, b) => sourceLabel(a).localeCompare(sourceLabel(b)), @@ -112,7 +115,7 @@ export default function BoardToolbar({ value={search} onChange={(e) => onSearchChange(e.target.value)} placeholder="Search name or ASIN" - aria-label="Search loaded products by name or ASIN" + aria-label="Search products by name or ASIN" spellCheck={false} />
@@ -128,7 +131,7 @@ export default function BoardToolbar({ type="button" className={clsx('board-seg__btn', view === 'movers' && 'board-seg__btn--on')} aria-pressed={view === 'movers'} - title="Biggest price drops across all sources" + title="Price changes between each source's latest two runs" onClick={() => onViewChange('movers')} > Movers @@ -150,7 +153,7 @@ export default function BoardToolbar({ ))} - +
); } diff --git a/dashboard/src/features/board/board.css b/dashboard/src/features/board/board.css index 311d9eb..b0b5fcb 100644 --- a/dashboard/src/features/board/board.css +++ b/dashboard/src/features/board/board.css @@ -303,3 +303,15 @@ .board-legend__note { color: var(--muted); } + +/* ── load more ─────────────────────────────────────────────────── */ +.board-more { + display: flex; + align-items: center; + gap: var(--space-3); + margin-top: var(--space-3); +} +.board-more__count { + color: var(--muted); + font-size: 12px; +} diff --git a/dashboard/src/features/board/sortRows.ts b/dashboard/src/features/board/sortRows.ts new file mode 100644 index 0000000..c056721 --- /dev/null +++ b/dashboard/src/features/board/sortRows.ts @@ -0,0 +1,32 @@ +// Sorts rows the way the board's table does, for rows the table never +// held (the full export). Uses the same accessors, with missing values last. + +import type { SortingState } from '@tanstack/react-table'; +import type { Product } from '../../lib/types'; +import { boardColumns } from './columns'; + +type Accessor = (p: Product) => unknown; + +function accessorOf(id: string): Accessor | null { + const col = boardColumns.find((c) => c.id === id) as { accessorFn?: (p: Product, i: number) => unknown } | undefined; + return col?.accessorFn ? (p) => col.accessorFn!(p, 0) : null; +} + +export function sortRows(rows: Product[], sorting: SortingState): Product[] { + const keys = sorting + .map((s) => ({ get: accessorOf(s.id), desc: s.desc })) + .filter((k): k is { get: Accessor; desc: boolean } => k.get !== null); + if (keys.length === 0) return rows; + return [...rows].sort((a, b) => { + for (const { get, desc } of keys) { + const x = get(a); + const y = get(b); + if (x === y) continue; + if (x === undefined || x === null) return 1; + if (y === undefined || y === null) return -1; + const cmp = typeof x === 'string' && typeof y === 'string' ? x.localeCompare(y) : (x as number) < (y as number) ? -1 : 1; + return desc ? -cmp : cmp; + } + return 0; + }); +} diff --git a/dashboard/src/features/drawer/HistoryDrawer.tsx b/dashboard/src/features/drawer/HistoryDrawer.tsx index bb6afc4..9d74063 100644 --- a/dashboard/src/features/drawer/HistoryDrawer.tsx +++ b/dashboard/src/features/drawer/HistoryDrawer.tsx @@ -12,6 +12,7 @@ import Drawer from '../../components/Drawer'; import EmptyState from '../../components/EmptyState'; import ErrorState from '../../components/ErrorState'; import { errorLabel } from '../../lib/errors'; +import SchemaNotice from '../../components/SchemaNotice'; import KeyValue from '../../components/KeyValue'; import Skeleton from '../../components/Skeleton'; import Sparkline, { type SparklinePoint } from '../../components/Sparkline'; @@ -149,6 +150,8 @@ export default function HistoryDrawer({ wid, asin, onClose }: HistoryDrawerProps ) : product.error ? ( + ) : product.invalid ? ( + ) : !p ? ( )}
-
+
{p.name ?? asin}
@@ -224,6 +227,8 @@ export default function HistoryDrawer({ wid, asin, onClose }: HistoryDrawerProps
{history.loading ? ( + ) : history.invalid ? ( + ) : history.error ? (
Couldn't load history: {errorLabel(history.error)} diff --git a/dashboard/src/features/drawer/data.ts b/dashboard/src/features/drawer/data.ts index 04e5669..486b33e 100644 --- a/dashboard/src/features/drawer/data.ts +++ b/dashboard/src/features/drawer/data.ts @@ -9,22 +9,13 @@ // NEVER listeners on history / snapshots. import { useEffect, useMemo, useState } from 'react'; -import { - doc, - setDoc, - type DocumentData, - type FirestoreDataConverter, -} from 'firebase/firestore'; +import { doc, setDoc } from 'firebase/firestore'; import { db } from '../../firebase'; +import { productConverter } from '../../lib/checked'; import { useDocOnce, type DocOnceState } from '../../lib/hooks'; import { latestOfferSnapshot } from '../../lib/queries'; import type { OfferSnapshot, Product, ProductLead } from '../../lib/types'; -const productConverter: FirestoreDataConverter = { - toFirestore: (data) => data as DocumentData, - fromFirestore: (snap) => snap.data() as Product, -}; - /** One-shot read of workspaces/{wid}/products/{asin}. Re-fetches only when * the doc path changes (useDocOnce keys on path). */ export function useProductOnce(wid: string, asin: string): DocOnceState { diff --git a/dashboard/src/features/export/ExportButton.tsx b/dashboard/src/features/export/ExportButton.tsx index fb8949f..0c66ced 100644 --- a/dashboard/src/features/export/ExportButton.tsx +++ b/dashboard/src/features/export/ExportButton.tsx @@ -1,6 +1,7 @@ -// Gold "Export XLSX" action for table views. Exports the already-loaded -// rows via exportProductsXlsx (which dynamic-imports xlsx on first click). -// Props contract is FROZEN: { rows, disabled? }. +// Gold "Export XLSX" action for table views. `load` fetches the rows to +// write, a page at a time when the scope is bigger than what is loaded, +// then exportProductsXlsx writes them (it dynamic-imports xlsx on first +// click). `count` is what the badge shows, when it is known up front. import { useEffect, useRef, useState } from 'react'; import Button from '../../components/Button'; @@ -10,7 +11,8 @@ import { reportError } from '../../lib/errors'; import './export.css'; export interface ExportButtonProps { - rows: Product[]; + load: (onProgress: (loaded: number) => void) => Promise; + count: number | null; disabled?: boolean; } @@ -18,8 +20,10 @@ type Phase = 'idle' | 'working' | 'done'; const CONFIRM_MS = 1500; -export default function ExportButton({ rows, disabled }: ExportButtonProps) { +export default function ExportButton({ load, count, disabled }: ExportButtonProps) { const [phase, setPhase] = useState('idle'); + const [progress, setProgress] = useState(0); + const [written, setWritten] = useState(0); const timerRef = useRef(null); const mountedRef = useRef(true); @@ -32,16 +36,21 @@ export default function ExportButton({ rows, disabled }: ExportButtonProps) { }, []); const handleClick = async () => { - if (phase === 'working' || rows.length === 0) return; + if (phase === 'working' || count === 0) return; if (timerRef.current !== null) { window.clearTimeout(timerRef.current); timerRef.current = null; } setPhase('working'); + setProgress(0); try { - // Async-aware: the first click also awaits the xlsx chunk download. + const rows = await load((n) => { + if (mountedRef.current) setProgress(n); + }); + if (rows.length === 0) throw new Error('nothing to export'); await exportProductsXlsx(rows); if (!mountedRef.current) return; + setWritten(rows.length); setPhase('done'); timerRef.current = window.setTimeout(() => { timerRef.current = null; @@ -53,7 +62,8 @@ export default function ExportButton({ rows, disabled }: ExportButtonProps) { } }; - const empty = rows.length === 0; + const empty = count === 0; + const fmt = (n: number) => n.toLocaleString('en-US'); return ( diff --git a/dashboard/src/features/export/exportXlsx.ts b/dashboard/src/features/export/exportXlsx.ts index 271c407..12ff775 100644 --- a/dashboard/src/features/export/exportXlsx.ts +++ b/dashboard/src/features/export/exportXlsx.ts @@ -1,6 +1,6 @@ -// XLSX export — writes the ALREADY-LOADED view rows to an FBA-lead-list -// workbook (zero extra Firestore reads — read-cost hygiene). The heavy -// `xlsx` (SheetJS) package is loaded via dynamic import() inside +// XLSX export — writes the rows ExportButton fetched (the whole scope, read +// a page at a time) to an FBA-lead-list workbook. The heavy `xlsx` +// (SheetJS) package is loaded via dynamic import() inside // exportProductsXlsx so it stays out of the main chunk; the row-shaping // logic below (buildRows / defaultFilename) is pure and testable. diff --git a/dashboard/src/features/runs/groupRuns.ts b/dashboard/src/features/runs/groupRuns.ts index de7c173..5460396 100644 --- a/dashboard/src/features/runs/groupRuns.ts +++ b/dashboard/src/features/runs/groupRuns.ts @@ -1,10 +1,11 @@ // Date-bucketing for the Run Inbox: Today / This week / Earlier. -// Buckets key off the run's UTC dayKey (falling back to startedAt) so the -// grouping matches the dayKey shown on each card. Input arrives newest- -// first from runsRecent(); bucketing preserves that order. +// Buckets key off the run's dayKey, the local date where it ran (falling +// back to startedAt), so the grouping matches the date on each card. +// Input arrives newest-first from runsNewestFirst(); bucketing keeps that +// order. import type { Run } from '../../lib/types'; -import { dayKey, dayKeyDiff, toMillis } from '../../lib/format'; +import { dayKeyDiff, localDayKey, toMillis } from '../../lib/format'; export type RunGroupKey = 'today' | 'week' | 'earlier'; @@ -20,15 +21,15 @@ export interface RunGroup { runs: Run[]; } -/** The run's UTC dayKey, derived from startedAt when absent. */ +/** The run's dayKey, derived from startedAt when absent. */ function runDayKey(run: Run): string | null { if (run.dayKey) return run.dayKey; const ms = toMillis(run.startedAt); - return ms === null ? null : dayKey(new Date(ms)); + return ms === null ? null : localDayKey(new Date(ms)); } -/** today = same UTC day; week = 1–6 days ago; earlier = older / undated. */ -export function groupKeyFor(run: Run, todayKey: string = dayKey()): RunGroupKey { +/** today = same local day; week = 1–6 days ago; earlier = older / undated. */ +export function groupKeyFor(run: Run, todayKey: string = localDayKey()): RunGroupKey { const key = runDayKey(run); if (!key) return 'earlier'; const daysAgo = dayKeyDiff(key, todayKey); @@ -39,7 +40,7 @@ export function groupKeyFor(run: Run, todayKey: string = dayKey()): RunGroupKey } /** Bucket runs (already newest-first) into ordered, non-empty groups. */ -export function groupRuns(runs: Run[], todayKey: string = dayKey()): RunGroup[] { +export function groupRuns(runs: Run[], todayKey: string = localDayKey()): RunGroup[] { const buckets: Record = { today: [], week: [], earlier: [] }; for (const run of runs) buckets[groupKeyFor(run, todayKey)].push(run); return (['today', 'week', 'earlier'] as const) diff --git a/dashboard/src/lib/checked.ts b/dashboard/src/lib/checked.ts new file mode 100644 index 0000000..5c43ed3 --- /dev/null +++ b/dashboard/src/lib/checked.ts @@ -0,0 +1,73 @@ +// Converters that check every document against the shared schema as it is +// read. A document that fails keeps its data but is marked, and the hooks +// leave it out of the view and count it, so a bad write shows up as a +// notice instead of a row of dashes. + +import type { + DocumentData, + FirestoreDataConverter, + QueryDocumentSnapshot, +} from 'firebase/firestore'; +import { + validateHistory, + validatePage, + validateProduct, + validateRun, + validateSource, +} from '../../../packages/schema/index.js'; +import type { HistoryDoc, PageDoc, Product, Run, Source } from './types'; + +export interface SchemaProblem { + path: string; + problems: string[]; +} + +const marks = new WeakMap(); + +/** The schema problems of a document read through a checked converter. */ +export function problemOf(value: object): SchemaProblem | undefined { + return marks.get(value); +} + +function checked( + validate: (doc: unknown) => string[], + // Documents the dashboard alone wrote (a lead note on a product the + // extension never saw, a nickname on a new source) carry no schema fields. + extensionWrote: (data: DocumentData) => boolean = () => true, +): FirestoreDataConverter { + return { + toFirestore: (data) => data as DocumentData, + fromFirestore(snap: QueryDocumentSnapshot) { + const data = snap.data(); + if (extensionWrote(data)) { + const problems = validate(data); + if (problems.length > 0) marks.set(data, { path: snap.ref.path, problems }); + } + return data as T; + }, + }; +} + +export const productConverter = checked( + validateProduct, + (d) => 'sv' in d || 'latest' in d, +); +export const runConverter = checked(validateRun); +export const pageConverter = checked(validatePage); +export const sourceConverter = checked(validateSource, (d) => 'sv' in d || 'lastRunId' in d); +export const historyConverter = checked(validateHistory, (d) => 'sv' in d); + +/** Splits rows into the ones that passed and the problems of the rest. */ +export function splitChecked(rows: T[]): { valid: T[]; invalid: SchemaProblem[] } { + const valid: T[] = []; + const invalid: SchemaProblem[] = []; + for (const row of rows) { + const problem = marks.get(row); + if (problem) invalid.push(problem); + else valid.push(row); + } + if (invalid.length > 0) { + console.warn(`[proscan] ${invalid.length} documents failed the schema check`, invalid.slice(0, 5)); + } + return { valid, invalid }; +} diff --git a/dashboard/src/lib/compare.ts b/dashboard/src/lib/compare.ts new file mode 100644 index 0000000..0d5a61d --- /dev/null +++ b/dashboard/src/lib/compare.ts @@ -0,0 +1,75 @@ +// Deltas and movers for one source, from its latest two runs. The page +// chunks of each run hold every ASIN the run saw with its point, so two +// runs of the same source compare like with like: the product document's +// own delta is against whatever run saw it last, from any source. +// +// Pure: page documents in, comparisons out. + +import { deltaOf, timeMs, type Delta, type Point } from '../../../packages/schema/index.js'; +import type { PageDoc, Product, Run } from './types'; + +const DAY_MS = 86400000; + +export interface Comparison { + asin: string; + now: Point; + /** null when the earlier run did not see this ASIN */ + prev: Point | null; + delta: Delta | null; +} + +/** Every ASIN in a run's pages with its point, first page first. */ +export function itemsOf(pages: Pick[]): Map { + const items = new Map(); + for (const page of [...pages].sort((a, b) => a.page - b.page)) { + for (const [asin, pt] of Object.entries(page.items)) { + if (items.has(asin)) continue; + const { sp: _sp, ...point } = pt; + items.set(asin, point); + } + } + return items; +} + +/** Whole days from the earlier run's start to the later one's. */ +export function daysBetween(latest: Pick, prev: Pick): number | null { + const a = timeMs(latest.startedAt); + const b = timeMs(prev.startedAt); + return a === null || b === null ? null : Math.floor((a - b) / DAY_MS); +} + +/** Each ASIN of the latest run against the earlier run. */ +export function compareRuns( + latestPages: Pick[], + prevPages: Pick[] | null, + days: number | null = null, +): Map { + const now = itemsOf(latestPages); + const before = prevPages ? itemsOf(prevPages) : new Map(); + const out = new Map(); + for (const [asin, pt] of now) { + const prev = before.get(asin) ?? null; + out.set(asin, { asin, now: pt, prev, delta: prev ? deltaOf(pt, prev, days) : null }); + } + return out; +} + +/** ASINs whose price changed, biggest drop first. */ +export function moversOf(comparisons: Iterable): Comparison[] { + return [...comparisons] + .filter((c) => c.delta?.p !== undefined && c.delta.p !== 0) + .sort((a, b) => (a.delta!.pPct ?? 0) - (b.delta!.pPct ?? 0) || a.delta!.p! - b.delta!.p!); +} + +/** + * A board row for a product as this source's latest run saw it: its point + * and its delta against the source's previous run. + */ +export function asSeenBy(product: Product, cmp: Comparison, run: Pick): Product { + return { + ...product, + latest: { ...cmp.now, at: run.startedAt, runId: run.runId, dayKey: run.dayKey }, + prev: cmp.prev, + delta: cmp.delta, + }; +} diff --git a/dashboard/src/lib/format.ts b/dashboard/src/lib/format.ts index 28c5be7..d2ef40a 100644 --- a/dashboard/src/lib/format.ts +++ b/dashboard/src/lib/format.ts @@ -2,6 +2,7 @@ // and timestamps. All money flows through here as integer cents. import { Timestamp } from 'firebase/firestore'; +import { dayKeyOf } from '../../../packages/schema/index.js'; import type { Source } from './types'; const MS_PER_DAY = 86_400_000; @@ -68,9 +69,9 @@ export function relativeTime(at: TimeLike): string { return future ? `in ${label}` : `${label} ago`; } -/** UTC 'YYYY-MM-DD' for a date (default: now). Matches run/history dayKeys. */ -export function dayKey(date: Date = new Date()): string { - return date.toISOString().slice(0, 10); +/** Local 'YYYY-MM-DD', the way the extension writes run and history dayKeys. */ +export function localDayKey(date: Date = new Date()): string { + return dayKeyOf(date.getTime(), date.getTimezoneOffset()); } /** 'YYYY-MM-DD' -> Date at UTC midnight. */ diff --git a/dashboard/src/lib/hooks.ts b/dashboard/src/lib/hooks.ts index 416ae5d..9ab18bf 100644 --- a/dashboard/src/lib/hooks.ts +++ b/dashboard/src/lib/hooks.ts @@ -10,6 +10,7 @@ import { createElement, useContext, useEffect, + useRef, useState, useSyncExternalStore, type DependencyList, @@ -25,6 +26,7 @@ import { type Query, } from 'firebase/firestore'; import { auth, db } from '../firebase'; +import { problemOf, splitChecked, type SchemaProblem } from './checked'; import type { Workspace } from './types'; /* ── scan activity signal ───────────────────────────────────────────── @@ -88,7 +90,10 @@ export function useAuthUser(): AuthState { /* ── snapshot query (live, scoped, self-detaching) ──────────────────── */ export interface SnapshotQueryState { + /** documents that passed the schema check */ data: T[]; + /** documents left out because they failed it */ + invalid: SchemaProblem[]; loading: boolean; error: Error | null; } @@ -104,6 +109,7 @@ export function useSnapshotQuery( ): SnapshotQueryState { const [state, setState] = useState>({ data: [], + invalid: [], loading: true, error: null, }); @@ -111,7 +117,7 @@ export function useSnapshotQuery( useEffect(() => { const q = queryFactory(); if (!q) { - setState({ data: [], loading: false, error: null }); + setState({ data: [], invalid: [], loading: false, error: null }); return; } setState((s) => ({ ...s, loading: true, error: null })); @@ -120,7 +126,10 @@ export function useSnapshotQuery( let mountReads = 0; const unsubscribe = onSnapshot( q, + { includeMetadataChanges: true }, (snap) => { + // An empty answer from the cache is not "nothing there" (F-48). + if (snap.empty && snap.metadata.fromCache) return; if (firstSnapshot) { firstSnapshot = false; scanEnd(token); @@ -132,11 +141,8 @@ export function useSnapshotQuery( `[proscan:reads] ${debugLabel ?? 'query'} +${delta} docs (mount total ${mountReads})`, ); } - setState({ - data: snap.docs.map((d) => d.data()), - loading: false, - error: null, - }); + const { valid, invalid } = splitChecked(snap.docs.map((d) => d.data() as T & object)); + setState({ data: valid, invalid, loading: false, error: null }); }, (error) => { if (firstSnapshot) { @@ -144,7 +150,7 @@ export function useSnapshotQuery( scanEnd(token); } console.error(`[proscan] ${debugLabel ?? 'query'} listener failed`, error); - setState({ data: [], loading: false, error }); + setState({ data: [], invalid: [], loading: false, error }); }, ); return () => { @@ -160,29 +166,42 @@ export function useSnapshotQuery( /* ── server-side count (aggregate, no documents read) ───────────────── */ /** getCountFromServer for a query. `null` while unknown or when the count - * failed; callers fall back to "showing first N". Re-runs on dep change. */ + * failed; callers fall back to "showing first N". Resets on dep change. + * A change of `refresh` (say, a live list's snapshot counter) recounts + * after a short pause and keeps the old number until the new one lands. */ export function useServerCount( queryFactory: () => Query | null, deps: DependencyList, + refresh: unknown = 0, ): number | null { const [count, setCount] = useState(null); + const fresh = useRef(true); useEffect(() => { - const q = queryFactory(); + fresh.current = true; setCount(null); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps); + useEffect(() => { + const q = queryFactory(); if (!q) return; let cancelled = false; - getCountFromServer(q) - .then((snap) => { - if (!cancelled) setCount(snap.data().count); - }) - .catch((error: unknown) => { - console.error('[proscan] count query failed', error); - }); + const delay = fresh.current ? 0 : 1000; + fresh.current = false; + const timer = setTimeout(() => { + getCountFromServer(q) + .then((snap) => { + if (!cancelled) setCount(snap.data().count); + }) + .catch((error: unknown) => { + console.error('[proscan] count query failed', error); + }); + }, delay); return () => { cancelled = true; + clearTimeout(timer); }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, deps); + }, [...deps, refresh]); return count; } @@ -190,6 +209,8 @@ export function useServerCount( export interface DocOnceState { data: T | null; + /** set when the document exists but failed the schema check */ + invalid: SchemaProblem | null; loading: boolean; error: Error | null; } @@ -199,6 +220,7 @@ export interface DocOnceState { export function useDocOnce(ref: DocumentReference | null): DocOnceState { const [state, setState] = useState>({ data: null, + invalid: null, loading: ref !== null, error: null, }); @@ -206,26 +228,25 @@ export function useDocOnce(ref: DocumentReference | null): DocOnceState useEffect(() => { if (!ref) { - setState({ data: null, loading: false, error: null }); + setState({ data: null, invalid: null, loading: false, error: null }); return; } let cancelled = false; - setState({ data: null, loading: true, error: null }); + setState({ data: null, invalid: null, loading: true, error: null }); getDoc(ref) .then((snap) => { if (cancelled) return; if (import.meta.env.DEV) { console.debug(`[proscan:reads] doc-once ${snap.ref.path} +1`); } - setState({ - data: snap.exists() ? snap.data() : null, - loading: false, - error: null, - }); + const data = snap.exists() ? snap.data() : null; + const invalid = data && typeof data === 'object' ? (problemOf(data) ?? null) : null; + if (invalid) console.warn('[proscan] document failed the schema check', invalid); + setState({ data: invalid ? null : data, invalid, loading: false, error: null }); }) .catch((error: Error) => { console.error(`[proscan] read ${ref.path} failed`, error); - if (!cancelled) setState({ data: null, loading: false, error }); + if (!cancelled) setState({ data: null, invalid: null, loading: false, error }); }); return () => { cancelled = true; diff --git a/dashboard/src/lib/movers.ts b/dashboard/src/lib/movers.ts new file mode 100644 index 0000000..43e7ba9 --- /dev/null +++ b/dashboard/src/lib/movers.ts @@ -0,0 +1,165 @@ +// Reads for run comparisons: a source's latest two run headers and their +// page chunks (one-shot, never listeners), and the product documents of +// the movers for names and images. Per source that is 2 + pages reads, +// plus one per mover shown. + +import { useEffect, useState } from 'react'; +import { getDocs } from 'firebase/firestore'; +import { splitChecked, type SchemaProblem } from './checked'; +import { asSeenBy, compareRuns, daysBetween, moversOf, type Comparison } from './compare'; +import { productsByAsin, runPages, runsOfSource } from './queries'; +import type { Product, Run, Source } from './types'; + +/** Sources compared when the board is not scoped to one. */ +export const MOVER_SOURCES = 8; +/** Movers joined to their product documents and shown. */ +export const MOVERS_SHOWN = 500; + +export interface SourceComparison { + sourceId: string; + latest: Run; + prev: Run | null; + byAsin: Map; +} + +async function compareSource( + wid: string, + sourceId: string, + invalid: SchemaProblem[], +): Promise { + const runsSnap = await getDocs(runsOfSource(wid, sourceId, 2)); + const runs = splitChecked(runsSnap.docs.map((d) => d.data())); + invalid.push(...runs.invalid); + const [latest, prev = null] = runs.valid; + if (!latest) return null; + const pagesOf = async (run: Run) => { + const snap = await getDocs(runPages(wid, run.runId)); + const pages = splitChecked(snap.docs.map((d) => d.data())); + invalid.push(...pages.invalid); + return pages.valid; + }; + const [latestPages, prevPages] = await Promise.all([pagesOf(latest), prev ? pagesOf(prev) : null]); + const days = prev ? daysBetween(latest, prev) : null; + return { sourceId, latest, prev, byAsin: compareRuns(latestPages, prevPages, days) }; +} + +interface State { + data: T; + invalid: SchemaProblem[]; + loading: boolean; + error: Error | null; +} + +/** The latest-two-runs comparison of each source in `sourceIds`. */ +export function useSourceComparisons( + wid: string | null, + sourceIds: string[] | null, +): State { + const key = sourceIds ? sourceIds.join('\n') : null; + const [state, setState] = useState>({ + data: [], + invalid: [], + loading: key !== null, + error: null, + }); + + useEffect(() => { + if (!wid || key === null) { + setState({ data: [], invalid: [], loading: false, error: null }); + return; + } + let cancelled = false; + setState({ data: [], invalid: [], loading: true, error: null }); + const invalid: SchemaProblem[] = []; + Promise.all(key.split('\n').filter(Boolean).map((id) => compareSource(wid, id, invalid))) + .then((all) => { + if (cancelled) return; + setState({ data: all.filter((c): c is SourceComparison => c !== null), invalid, loading: false, error: null }); + }) + .catch((error: Error) => { + console.error('[proscan] run comparison failed', error); + if (!cancelled) setState({ data: [], invalid, loading: false, error }); + }); + return () => { + cancelled = true; + }; + }, [wid, key]); + + return state; +} + +/** The sources Movers compares: the one in scope, else the most recently scanned. */ +export function moverSources(sourceId: string | null, sources: Source[]): string[] { + if (sourceId) return [sourceId]; + return [...sources] + .filter((s) => s.lastRunId) + .sort((a, b) => (b.lastScrapedAt?.toMillis() ?? 0) - (a.lastScrapedAt?.toMillis() ?? 0)) + .slice(0, MOVER_SOURCES) + .map((s) => s.sourceId); +} + +export interface MoverRows { + rows: Product[]; + /** price changes found, before the MOVERS_SHOWN cap */ + found: number; + compared: SourceComparison[]; +} + +/** + * Movers across the given comparisons, biggest drop first, as board rows: + * each ASIN as its source's latest run saw it. An ASIN in several sources + * shows under the most recently scanned one. + */ +export function useMoverRows(wid: string | null, comparisons: State): State { + const [state, setState] = useState>({ + data: { rows: [], found: 0, compared: [] }, + invalid: [], + loading: true, + error: null, + }); + + useEffect(() => { + if (!wid || comparisons.loading || comparisons.error) { + setState({ data: { rows: [], found: 0, compared: [] }, invalid: [], loading: comparisons.loading, error: comparisons.error }); + return; + } + const picked = new Map(); + for (const sc of comparisons.data) { + for (const cmp of moversOf(sc.byAsin.values())) { + if (!picked.has(cmp.asin)) picked.set(cmp.asin, { cmp, run: sc.latest }); + } + } + const ordered = moversOf([...picked.values()].map((p) => p.cmp)); + const shown = ordered.slice(0, MOVERS_SHOWN); + let cancelled = false; + setState((s) => ({ ...s, loading: true, error: null })); + + const chunks: string[][] = []; + for (let i = 0; i < shown.length; i += 30) chunks.push(shown.slice(i, i + 30).map((c) => c.asin)); + Promise.all(chunks.map((asins) => getDocs(productsByAsin(wid, asins)))) + .then((snaps) => { + if (cancelled) return; + const { valid, invalid } = splitChecked(snaps.flatMap((s) => s.docs.map((d) => d.data()))); + const docs = new Map(valid.map((p) => [p.asin, p])); + const rows = shown.map((cmp) => { + const product = docs.get(cmp.asin) ?? { asin: cmp.asin, mk: 'US' }; + return asSeenBy(product, cmp, picked.get(cmp.asin)!.run); + }); + setState({ + data: { rows, found: ordered.length, compared: comparisons.data }, + invalid: [...comparisons.invalid, ...invalid], + loading: false, + error: null, + }); + }) + .catch((error: Error) => { + console.error('[proscan] mover products failed', error); + if (!cancelled) setState({ data: { rows: [], found: 0, compared: [] }, invalid: [], loading: false, error }); + }); + return () => { + cancelled = true; + }; + }, [wid, comparisons]); + + return state; +} diff --git a/dashboard/src/lib/paging.ts b/dashboard/src/lib/paging.ts new file mode 100644 index 0000000..c374476 --- /dev/null +++ b/dashboard/src/lib/paging.ts @@ -0,0 +1,147 @@ +// Pagination over an ordered query. One live listener covers every page +// loaded so far (limit(size * pages)), so a scan that is syncing shows up +// without a reload and a row pushed down past a page boundary is never +// dropped between pages. "Load more" grows the listener by one page, which +// re-reads what is already loaded; that is the price of never losing rows. +// Nothing is ever capped silently: `hasMore` says when the server has more. + +import { useEffect, useState, type DependencyList } from 'react'; +import { + getDocs, + limit, + onSnapshot, + query, + startAfter, + type Query, + type QueryDocumentSnapshot, +} from 'firebase/firestore'; +import { splitChecked, type SchemaProblem } from './checked'; + +export const PAGE_SIZE = 200; + +export interface PagedState { + data: T[]; + invalid: SchemaProblem[]; + loading: boolean; + error: Error | null; + hasMore: boolean; + loadingMore: boolean; + loadMore: () => void; + /** Bumps on every server snapshot that changed documents. */ + changes: number; +} + +interface Page { + rows: T[]; + invalid: SchemaProblem[]; + last: QueryDocumentSnapshot | null; + full: boolean; +} + +function toPage(docs: QueryDocumentSnapshot[], size: number): Page { + const { valid, invalid } = splitChecked(docs.map((d) => d.data() as T & object)); + return { rows: valid, invalid, last: docs[docs.length - 1] ?? null, full: docs.length >= size }; +} + +const sameDeps = (a: DependencyList, b: DependencyList) => + a.length === b.length && a.every((v, i) => Object.is(v, b[i])); + +interface Shown extends Page { + deps: DependencyList; + pages: number; +} + +/** Pages through `queryFactory()` (ordered, no limit) `size` documents at a time. */ +export function usePagedQuery( + queryFactory: () => Query | null, + deps: DependencyList, + size: number = PAGE_SIZE, +): PagedState { + // Pages wanted for these deps; a deps change starts again at one page. + const [grow, setGrow] = useState({ deps, pages: 1 }); + const pages = sameDeps(grow.deps, deps) ? grow.pages : 1; + const [shownRaw, setShown] = useState | null>(null); + const shown = shownRaw && sameDeps(shownRaw.deps, deps) ? shownRaw : null; + const [error, setError] = useState<{ deps: DependencyList; err: Error } | null>(null); + const [changes, setChanges] = useState(0); + const [active, setActive] = useState(true); + + useEffect(() => { + const q = queryFactory(); + setError(null); + setActive(q !== null); + if (!q) return; + const want = size * pages; + let fromServer = false; + // Metadata changes too, so the server's answer arrives even when it + // matches an empty cache (F-48). A grown listener also skips its cache + // answer, which would briefly show fewer rows than were loaded. + return onSnapshot( + query(q, limit(want)), + { includeMetadataChanges: true }, + (snap) => { + const cached = snap.metadata.fromCache; + if (cached && !fromServer && (snap.empty || pages > 1)) return; + if (!cached) fromServer = true; + setShown({ ...toPage(snap.docs, want), deps, pages }); + if (!cached && snap.docChanges().length > 0) setChanges((n) => n + 1); + }, + (err) => { + console.error('[proscan] paged query failed', err); + setError({ deps, err }); + }, + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [...deps, pages, size]); + + const hasMore = !!shown && shown.full; + const loadingMore = !!shown && shown.pages < pages; + const loadMore = () => { + if (hasMore && !loadingMore) setGrow({ deps, pages: pages + 1 }); + }; + const err = error && sameDeps(error.deps, deps) ? error.err : null; + + return { + data: shown?.rows ?? [], + invalid: shown?.invalid ?? [], + loading: !err && active && shown === null, + error: err, + hasMore, + loadingMore, + loadMore, + changes, + }; +} + +/** The count to show for a paged list. When everything is loaded that is + * the loaded rows (invalid documents are reported apart). Otherwise it is + * the server count minus the invalid documents seen, and never below what + * is already on screen, since the count can lag a live sync. */ +export function pagedTotal(state: PagedState, serverCount: number | null): number | null { + if (state.loading) return serverCount; + if (!state.hasMore) return state.data.length; + if (serverCount === null) return null; + return Math.max(serverCount - state.invalid.length, state.data.length); +} + +/** Every document of `q`, read `size` at a time with a cursor. For export. */ +export async function fetchAll( + q: Query, + onProgress?: (loaded: number) => void, + size = 500, +): Promise<{ rows: T[]; invalid: SchemaProblem[] }> { + const rows: T[] = []; + const invalid: SchemaProblem[] = []; + let cursor: QueryDocumentSnapshot | null = null; + for (;;) { + const page: Query = cursor ? query(q, startAfter(cursor), limit(size)) : query(q, limit(size)); + const snap = await getDocs(page); + const part: Page = toPage(snap.docs, size); + rows.push(...part.rows); + invalid.push(...part.invalid); + onProgress?.(rows.length); + if (!part.full || !part.last) break; + cursor = part.last; + } + return { rows, invalid }; +} diff --git a/dashboard/src/lib/queries.ts b/dashboard/src/lib/queries.ts index 0afb005..7ddc129 100644 --- a/dashboard/src/lib/queries.ts +++ b/dashboard/src/lib/queries.ts @@ -3,13 +3,14 @@ // auditable in one file (docs/ops/billing-runbook.md): // * products are ALWAYS scoped (per-source / top-N / recent-N) — never // the whole collection; -// * runs list is capped at 30; +// * runs are paged 30 at a time; // * sources is the only whole-collection read (tiny by design); // * history + offerSnapshots are one-shot fetch material. import { collection, doc, + documentId, getDocs, limit, orderBy, @@ -23,10 +24,18 @@ import { type Query, } from 'firebase/firestore'; import { db } from '../firebase'; +import { + historyConverter, + pageConverter, + productConverter, + runConverter, + sourceConverter, +} from './checked'; import type { HistoryDoc, LeadStage, OfferSnapshot, + PageDoc, Product, ProductEvent, ProductLead, @@ -34,20 +43,13 @@ import type { Source, } from './types'; -/* ── converters (id-stamping, cast-only — schema lives in types.ts) ─── */ - -function converter(): FirestoreDataConverter { - return { - toFirestore: (data) => data as DocumentData, - fromFirestore: (snap) => snap.data() as T, - }; -} +/* ── converters: checked against the shared schema (lib/checked.ts) ─── */ -const productConverter = converter(); -const runConverter = converter(); -const sourceConverter = converter(); -const historyConverter = converter(); -const snapshotConverter = converter(); +// Offer snapshots are Phase 5 and not in the schema yet. +const snapshotConverter: FirestoreDataConverter = { + toFirestore: (data) => data as DocumentData, + fromFirestore: (snap) => snap.data() as OfferSnapshot, +}; function productsCol(wid: string) { return collection(db, 'workspaces', wid, 'products').withConverter( @@ -75,30 +77,47 @@ export function productsBySource( return lim === null ? q : query(q, limit(lim)); } -/** Global Movers / Flip Radar: biggest price DROPS first (delta.pPct asc — - * most-negative = best buying opportunity). Single-field index. */ -export function topMovers(wid: string, lim: number | null = 100): Query { - const q = query(productsCol(wid), orderBy('delta.pPct', 'asc')); - return lim === null ? q : query(q, limit(lim)); -} - /** Most recently observed products across all sources. */ export function recentProducts(wid: string, lim: number | null = 200): Query { const q = query(productsCol(wid), orderBy('latest.at', 'desc')); return lim === null ? q : query(q, limit(lim)); } +/** One product by ASIN, for exact search past the loaded pages. */ +export function productRef(wid: string, asin: string): DocumentReference { + return doc(db, 'workspaces', wid, 'products', asin).withConverter(productConverter); +} + /* ── runs / sources ─────────────────────────────────────────────────── */ -/** Run Inbox — newest first, HARD-capped at 30 (read hygiene). */ -export function runsRecent(wid: string, lim = 30): Query { +/** Run Inbox — newest first, no limit: page it with usePagedQuery. */ +export function runsNewestFirst(wid: string): Query { + return query( + collection(db, 'workspaces', wid, 'runs').withConverter(runConverter), + orderBy('startedAt', 'desc'), + ); +} + +/** A source's newest runs. Index: runs(sourceId asc, startedAt desc). */ +export function runsOfSource(wid: string, sourceId: string, lim: number): Query { return query( collection(db, 'workspaces', wid, 'runs').withConverter(runConverter), + where('sourceId', '==', sourceId), orderBy('startedAt', 'desc'), - limit(Math.min(lim, 30)), + limit(lim), ); } +/** A run's page chunks, one-shot material for run comparisons. */ +export function runPages(wid: string, runId: string): Query { + return collection(db, 'workspaces', wid, 'runs', runId, 'pages').withConverter(pageConverter); +} + +/** Up to 30 products by ASIN (the `in` limit). */ +export function productsByAsin(wid: string, asins: string[]): Query { + return query(productsCol(wid), where(documentId(), 'in', asins.slice(0, 30))); +} + /** Whole sources collection (the watchlist spine — tiny by design; * staleness sorting happens client-side via format.staleness). */ export function sources(wid: string): Query { diff --git a/dashboard/src/lib/types.ts b/dashboard/src/lib/types.ts index dafccf7..f77c045 100644 --- a/dashboard/src/lib/types.ts +++ b/dashboard/src/lib/types.ts @@ -1,12 +1,15 @@ -// ProScan data contract — TypeScript mirrors of the Firestore schema in -// docs/architecture/data-model.md. All money fields are INTEGER CENTS; -// format exclusively via lib/format.ts. Compact point keys are shared by -// latest / prev / history.d values / page items. +// Dashboard types. The extension-written shapes come from the shared +// schema (packages/schema, vendored from the extension); this file adds +// the dashboard-owned fields (lead, tags, nickname, cadence) and the +// Phase 5 spread fields nothing writes yet. Money is integer cents; format +// it through lib/format.ts. import type { Timestamp } from 'firebase/firestore'; +import type { PageDoc as SchemaPageDoc, RunDoc } from '../../../packages/schema/index.js'; /** Compact observation point. Every field may be absent (e.g. `p` is - * omitted when the price parse failed). Money fields are cents. */ + * omitted when the price parse failed). Money fields are cents. p, r, v, + * pr, rk and sp are schema v1; the rest are reserved for Phase 5. */ export interface Point { /** price ¢ */ p?: number; @@ -134,14 +137,15 @@ export interface ProductLead { /** Canonical ASIN doc — workspaces/{wid}/products/{asin}. * spread / scores / verdict / delta / lead MAY BE ABSENT. */ export interface Product { + sv?: number; asin: string; mk: string; - name?: string; - img?: string; + name?: string | null; + img?: string | null; url?: string; latest?: ObservedPoint; - prev?: ObservedPoint; - delta?: ProductDelta; + prev?: ObservedPoint | null; + delta?: ProductDelta | null; spread?: ProductSpread; scores?: ProductScores; verdict?: ProductVerdict; @@ -162,42 +166,28 @@ export interface RunCounters { newSeen?: number; } -/** Scrape-run header — workspaces/{wid}/runs/{runId}. */ -export interface Run { - runId: string; - sourceId: string; - source?: { - type?: SourceType; - sellerId?: string | null; - keyword?: string | null; - url?: string; - }; - mk?: string; - /** UTC date of startedAt, 'YYYY-MM-DD' */ - dayKey?: string; - startedAt?: Timestamp; - /** null while active */ - finishedAt?: Timestamp | null; - status: RunStatus; - pagesDone?: number; - pagesPlanned?: number; - totalResultsOnSerp?: number; - counters?: RunCounters; +/** Scrape-run header — workspaces/{wid}/runs/{runId}. dayKey is the + * local date where the scan ran. */ +export type Run = RunDoc & { /** user-renamable run card */ label?: string | null; -} +}; + +/** One page of a run — workspaces/{wid}/runs/{runId}/pages/{pageId}. */ +export type PageDoc = SchemaPageDoc; export type SourceType = 'storefront' | 'keyword'; /** Watchlist entry — workspaces/{wid}/sources/{sourceId}. */ export interface Source { + sv?: number; sourceId: string; type: SourceType; /** null for keyword sources */ sellerId?: string | null; keyword?: string | null; nickname?: string | null; - url?: string; + url?: string | null; watched?: boolean; /** Rescan Queue: staleness = now − lastScrapedAt vs cadence */ cadenceDays?: number; @@ -210,8 +200,9 @@ export interface Source { } /** Date-keyed time-series — products/{asin}/history/daily. - * One Point per UTC dayKey; spread-day extras merged into the same entry. */ + * One Point per dayKey; spread-day extras merged into the same entry. */ export interface HistoryDoc { + sv?: number; asin: string; d: Record; } diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index d0d96e1..c6fbfa4 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -2,15 +2,16 @@ // Excel-killer. URL contract (FROZEN): ?source= scopes the board // to one source; ?asin= opens the HistoryDrawer. // -// Read hygiene (docs/ops/billing-runbook.md): exactly ONE scoped products -// listener at a time — -// movers -> topMovers(wid, 100) (global biggest drops) -// source -> productsBySource(wid, sourceId, 500) -// default -> recentProducts(wid, 300) -// plus the sanctioned tiny sources listener for the scope dropdown. -// Search, sorting, export and the movers' has-delta filter are client-side -// over the loaded set only, so the header says when that set is capped and -// shows the server-side total (F-45). Real pagination lands in Phase 4. +// Reads (docs/ops/billing-runbook.md): +// latest -> productsBySource / recentProducts, 200 at a time: the first +// page is live, "Load more" pages on with a cursor (F-45) +// total -> getCountFromServer on the same query, so the header always +// says how many there are, not how many are loaded +// movers -> each source's latest two runs and their page chunks +// (lib/movers.ts): price changes run against run, per source +// plus the sanctioned tiny sources listener for the scope dropdown. With a +// source in scope, Latest also shows deltas against that source's previous +// run. An exact ASIN in the search box is looked up directly. import { useMemo, useState, type ReactNode } from 'react'; import { useSearchParams } from 'react-router-dom'; @@ -20,17 +21,23 @@ import { useReactTable, type SortingState, } from '@tanstack/react-table'; -import { useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { useDocOnce, useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { fetchAll, pagedTotal, usePagedQuery, PAGE_SIZE } from '../lib/paging'; +import { reportError } from '../lib/errors'; import { + productRef, productsBySource, recentProducts, sources as sourcesQuery, - topMovers, } from '../lib/queries'; +import { asSeenBy } from '../lib/compare'; +import { MOVERS_SHOWN, moverSources, useMoverRows, useSourceComparisons } from '../lib/movers'; +import { ASIN_RE } from '../../../packages/schema/index.js'; import type { Product } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Skeleton from '../components/Skeleton'; import Button from '../components/Button'; import HistoryDrawer from '../features/drawer/HistoryDrawer'; @@ -41,28 +48,26 @@ import BoardToolbar, { } from '../features/board/BoardToolbar'; import BoardTable from '../features/board/BoardTable'; import { boardColumns, dataColumnVisibility } from '../features/board/columns'; +import { sortRows } from '../features/board/sortRows'; import '../features/board/board.css'; -const LIMITS = { movers: 100, source: 500, recent: 300 } as const; - const fmt = (n: number) => n.toLocaleString('en-US'); -/** Header count. Never presents a capped window as the whole workspace. */ -function countLabel( +/** Header count: how many there are, and how many of them are loaded. */ +export function countLabel( visible: number, loaded: number, - capped: boolean, total: number | null, + searching: boolean, ): string { - const noun = loaded === 1 ? 'product' : 'products'; - if (!capped || (total !== null && total <= loaded)) { - return visible === loaded ? `${fmt(loaded)} ${noun}` : `${fmt(visible)} of ${fmt(loaded)} ${noun}`; + const noun = (n: number) => (n === 1 ? 'product' : 'products'); + const all = total !== null && loaded >= total; + if (searching) { + const scope = all ? fmt(loaded) : `${fmt(loaded)} loaded`; + return `${fmt(visible)} ${visible === 1 ? 'match' : 'matches'} in ${scope} ${noun(loaded)}`; } - const span = - total !== null - ? `first ${fmt(loaded)} of ${fmt(total)} ${noun}` - : `first ${fmt(loaded)} ${noun}, more not loaded`; - return visible === loaded ? `Showing the ${span}` : `${fmt(visible)} matches in the ${span}`; + if (all || total === null) return `${fmt(total ?? loaded)} ${noun(total ?? loaded)}`; + return `${fmt(loaded)} of ${fmt(total)} ${noun(total)} loaded`; } export default function Products() { @@ -78,31 +83,21 @@ export default function Products() { const [search, setSearch] = useState(''); const [sorting, setSorting] = useState([]); - const limit = - view === 'movers' ? LIMITS.movers : sourceId ? LIMITS.source : LIMITS.recent; + const scopeQuery = () => { + if (!wid) return null; + return sourceId ? productsBySource(wid, sourceId, null) : recentProducts(wid, null); + }; - const products = useSnapshotQuery( - () => { - if (!wid) return null; - if (view === 'movers') return topMovers(wid, limit); - if (sourceId) return productsBySource(wid, sourceId, limit); - return recentProducts(wid, limit); - }, + const latest = usePagedQuery( + () => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId], - view === 'movers' ? 'board:movers' : sourceId ? 'board:by-source' : 'board:recent', ); - - // Only ask the server for a total once the window is actually full. - const capped = products.data.length >= limit; - const total = useServerCount( - () => { - if (!wid || !capped) return null; - if (view === 'movers') return topMovers(wid, null); - if (sourceId) return productsBySource(wid, sourceId, null); - return recentProducts(wid, null); - }, - [wid, view, sourceId, capped], + const serverCount = useServerCount( + () => (view === 'latest' ? scopeQuery() : null), + [wid, view, sourceId], + latest.changes, ); + const total = pagedTotal(latest, serverCount); const sourcesState = useSnapshotQuery( () => (wid ? sourcesQuery(wid) : null), @@ -110,29 +105,64 @@ export default function Products() { 'board:sources', ); - // Movers shows only rows that actually carry a delta; a source scope in - // movers view narrows client-side (the movers query is global). - const scopedRows = useMemo(() => { - if (view !== 'movers') return products.data; - let list = products.data.filter( - (p) => p.delta?.pPct !== undefined || p.delta?.p !== undefined, - ); - if (sourceId) list = list.filter((p) => p.sourceIds?.includes(sourceId)); - return list; - }, [products.data, view, sourceId]); + // Which sources to compare run against run: the one in scope, or for + // Movers across all sources the most recently scanned ones. + const compareIds = useMemo(() => { + if (view === 'movers') { + if (!sourceId && sourcesState.loading) return null; + return moverSources(sourceId, sourcesState.data); + } + return sourceId ? [sourceId] : null; + }, [view, sourceId, sourcesState.loading, sourcesState.data]); + const comparisons = useSourceComparisons(wid, compareIds); + const movers = useMoverRows(view === 'movers' ? wid : null, comparisons); + + // In a source's scope, Latest shows each product as that source's last + // run saw it, with the delta against the run before. + const overlay = view === 'latest' && sourceId ? comparisons.data[0] : undefined; + const latestRows = useMemo(() => { + if (!overlay) return latest.data; + return latest.data.map((p) => { + const cmp = overlay.byAsin.get(p.asin); + return cmp ? asSeenBy(p, cmp, overlay.latest) : p; + }); + }, [latest.data, overlay]); - // Search-in-loaded-set: name or ASIN, case-insensitive. + const scopedRows = view === 'movers' ? movers.data.rows : latestRows; + const loaded = scopedRows; + const state = + view === 'movers' + ? movers + : { + ...latest, + loading: latest.loading || (sourceId !== null && comparisons.loading), + error: latest.error ?? comparisons.error, + invalid: [...latest.invalid, ...comparisons.invalid], + }; + + // An exact ASIN is read directly, so search reaches past the loaded pages. const query = search.trim().toLowerCase(); + const exactAsin = ASIN_RE.test(search.trim().toUpperCase()) ? search.trim().toUpperCase() : null; + const exactRef = useMemo( + () => (wid && exactAsin ? productRef(wid, exactAsin) : null), + [wid, exactAsin], + ); + const exact = useDocOnce(exactRef); + const rows = useMemo(() => { if (!query) return scopedRows; - return scopedRows.filter( + const hits = scopedRows.filter( (p) => p.asin.toLowerCase().includes(query) || (p.name ?? '').toLowerCase().includes(query), ); - }, [scopedRows, query]); + const found = exact.data; + const inScope = found && (!sourceId || found.sourceIds?.includes(sourceId)); + if (found && inScope && !hits.some((p) => p.asin === found.asin)) return [found, ...hits]; + return hits; + }, [scopedRows, query, exact.data, sourceId]); - const columnVisibility = useMemo(() => dataColumnVisibility(products.data), [products.data]); + const columnVisibility = useMemo(() => dataColumnVisibility(loaded), [loaded]); const table = useReactTable({ data: rows, @@ -144,8 +174,26 @@ export default function Products() { getRowId: (p) => p.asin, }); - // What the user currently sees, in sorted order — the export contract. - const visibleSortedRows = table.getRowModel().rows.map((r) => r.original); + // Export writes the whole scope, not the loaded pages: every product, + // searched and sorted like the table. Movers are all loaded already. + const matches = (p: Product) => + !query || p.asin.toLowerCase().includes(query) || (p.name ?? '').toLowerCase().includes(query); + const exportLoad = async (onProgress: (n: number) => void): Promise => { + const base = scopeQuery(); + if (view === 'movers' || !base || !latest.hasMore) return table.getRowModel().rows.map((r) => r.original); + const { rows: all, invalid } = await fetchAll(base, onProgress); + if (invalid.length > 0) { + reportError('export some rows', new Error(`${invalid.length} failed the schema check and were left out`)); + } + const seen = overlay + ? all.map((p) => { + const cmp = overlay.byAsin.get(p.asin); + return cmp ? asSeenBy(p, cmp, overlay.latest) : p; + }) + : all; + return sortRows(seen.filter(matches), sorting); + }; + const exportCount = view === 'movers' || !latest.hasMore ? rows.length : query ? null : total; /* ── URL writers ──────────────────────────────────────────────── */ @@ -176,17 +224,29 @@ export default function Products() { return s ? sourceLabel(s) : sourceId; }, [sourceId, sourcesState.data]); - const loading = !wid || products.loading; + const loading = !wid || state.loading; + + const moversLine = () => { + const { found, compared } = movers.data; + const n = found > MOVERS_SHOWN ? `top ${fmt(MOVERS_SHOWN)} of ${fmt(found)} movers` : `${fmt(found)} ${found === 1 ? 'mover' : 'movers'}`; + const only = compared.length === 1 ? compared[0] : null; + if (only) { + return only.prev + ? `${n} · run of ${only.latest.dayKey} against ${only.prev.dayKey}` + : `${n} · only one run so far`; + } + return `${n} across ${compared.length} ${compared.length === 1 ? 'source' : 'sources'}, each run against the one before`; + }; const subtitle = loading ? 'Loading the board…' : [ - countLabel(rows.length, scopedRows.length, capped && !(view === 'movers' && sourceId), total), + view === 'movers' ? moversLine() : countLabel(rows.length, scopedRows.length, total, query !== ''), view === 'movers' - ? sourceId - ? `biggest drops within the global top ${limit}` - : 'biggest price drops first' - : 'latest observations', + ? 'biggest price drops first' + : overlay?.prev + ? `deltas against the run of ${overlay.prev.dayKey}` + : 'latest observations', sourceName ? `source: ${sourceName}` : null, ] .filter(Boolean) @@ -201,9 +261,9 @@ export default function Products() {
); - } else if (products.error) { + } else if (state.error) { body = ( - + ); } else if (scopedRows.length === 0) { if (view === 'movers') { @@ -247,7 +307,11 @@ export default function Products() { body = ( setSearch('')}> Clear search @@ -259,6 +323,8 @@ export default function Products() { body = ; } + const showMore = view === 'latest' && !loading && !state.error && latest.hasMore; + return ( <> } /> + {body} + {showMore ? ( +
+ + + {total !== null + ? `${fmt(latest.data.length)} of ${fmt(total)} loaded` + : `${fmt(latest.data.length)} loaded`} + +
+ ) : null} {asin && wid ? ( ) : null} diff --git a/dashboard/src/routes/Runs.tsx b/dashboard/src/routes/Runs.tsx index 55866f6..499af62 100644 --- a/dashboard/src/routes/Runs.tsx +++ b/dashboard/src/routes/Runs.tsx @@ -1,31 +1,40 @@ -// Run Inbox (MVP feature 2) — the 30 most recent scrape-run headers as -// instrument cards, newest first, grouped Today / This week / Earlier. +// Run Inbox (MVP feature 2) — scrape-run headers as instrument cards, +// newest first, 30 at a time, grouped Today / This week / Earlier. // Source names are joined client-side from the (tiny, sanctioned) sources // listener. Clicking a card opens the product board scoped to that // run's source: /?source=. import { Fragment, useMemo } from 'react'; import { useNavigate } from 'react-router-dom'; -import { useSnapshotQuery, useWorkspace } from '../lib/hooks'; -import { runsRecent, sources } from '../lib/queries'; +import { useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { pagedTotal, usePagedQuery } from '../lib/paging'; +import { runsNewestFirst, sources } from '../lib/queries'; import type { Source } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Button from '../components/Button'; import { CWS_URL } from '../auth/SignIn'; import RunCard, { RunCardSkeleton } from '../features/runs/RunCard'; import { groupRuns } from '../features/runs/groupRuns'; import '../features/runs/runs.css'; +import '../features/board/board.css'; + +const RUNS_PAGE = 30; export default function Runs() { const { wid } = useWorkspace(); const navigate = useNavigate(); - const runsState = useSnapshotQuery( - () => (wid ? runsRecent(wid, 30) : null), + const runsState = usePagedQuery( + () => (wid ? runsNewestFirst(wid) : null), [wid], - 'runs:recent30', + RUNS_PAGE, + ); + const total = pagedTotal( + runsState, + useServerCount(() => (wid ? runsNewestFirst(wid) : null), [wid], runsState.changes), ); const sourcesState = useSnapshotQuery( () => (wid ? sources(wid) : null), @@ -95,9 +104,25 @@ export default function Runs() { <> + {body} + {!loading && !runsState.error && runsState.hasMore ? ( +
+ + + {runsState.data.length} + {total !== null ? ` of ${total}` : ''} loaded + +
+ ) : null} ); } diff --git a/dashboard/src/routes/Watchlist.tsx b/dashboard/src/routes/Watchlist.tsx index 54b01a6..01ee28a 100644 --- a/dashboard/src/routes/Watchlist.tsx +++ b/dashboard/src/routes/Watchlist.tsx @@ -9,6 +9,7 @@ import { staleness } from '../lib/format'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Skeleton from '../components/Skeleton'; import Button from '../components/Button'; import { CWS_URL } from '../auth/SignIn'; @@ -21,7 +22,7 @@ function plural(n: number, word: string): string { export default function Watchlist() { const { wid } = useWorkspace(); - const { data, loading, error } = useSnapshotQuery( + const { data, invalid, loading, error } = useSnapshotQuery( () => (wid ? sources(wid) : null), [wid], 'watchlist:sources', @@ -74,6 +75,7 @@ export default function Watchlist() { subtitle="Storefronts and keywords on a rescan cadence — fresh within cadence, due past it, stale past twice it." actions={summary ? {summary} : undefined} /> + {body} ); diff --git a/dashboard/src/shell/Layout.tsx b/dashboard/src/shell/Layout.tsx index 58992e1..de5d69a 100644 --- a/dashboard/src/shell/Layout.tsx +++ b/dashboard/src/shell/Layout.tsx @@ -2,6 +2,7 @@ import { Outlet, useLocation } from 'react-router-dom'; import Sidebar from './Sidebar'; import Topbar from './Topbar'; import Toaster from '../components/Toaster'; +import ExtensionPassword from '../auth/ExtensionPassword'; import './shell.css'; /** App shell grid: 232px chrome sidebar + topbar + light content area. @@ -16,6 +17,7 @@ export default function Layout() {
+
diff --git a/dashboard/test/compare.test.mjs b/dashboard/test/compare.test.mjs new file mode 100644 index 0000000..49dfb82 --- /dev/null +++ b/dashboard/test/compare.test.mjs @@ -0,0 +1,53 @@ +// Unit tests for lib/compare.ts. Run: npm run test:unit +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { asSeenBy, compareRuns, daysBetween, itemsOf, moversOf } from '../src/lib/compare.ts'; + +const page = (n, items) => ({ page: n, items }); + +test('itemsOf keeps the first page an ASIN was on and drops the sponsored flag', () => { + const items = itemsOf([page(2, { B000000001: { p: 900, sp: 1 } }), page(1, { B000000001: { p: 1000, rk: 3, sp: 0 } })]); + assert.deepEqual(items.get('B000000001'), { p: 1000, rk: 3 }); +}); + +test('compareRuns diffs each ASIN of the latest run against the earlier one', () => { + const latest = [page(1, { B000000001: { p: 900, r: 4.5, v: 110 }, B000000002: { p: 500 } })]; + const prev = [page(1, { B000000001: { p: 1000, r: 4.5, v: 100 } })]; + const cmp = compareRuns(latest, prev, 7); + assert.deepEqual(cmp.get('B000000001').delta, { p: -100, pPct: -10, r: 0, v: 10, days: 7 }); + assert.equal(cmp.get('B000000002').prev, null, 'new to this source'); + assert.equal(cmp.get('B000000002').delta, null); +}); + +test('a price missing from either run gives no price delta, not a zero', () => { + const cmp = compareRuns([page(1, { B000000001: { v: 12 } })], [page(1, { B000000001: { p: 1000, v: 10 } })]); + assert.deepEqual(cmp.get('B000000001').delta, { v: 2 }); + assert.deepEqual(moversOf(cmp.values()), []); +}); + +test('with no earlier run nothing has a delta', () => { + const cmp = compareRuns([page(1, { B000000001: { p: 1 } })], null); + assert.equal(cmp.get('B000000001').delta, null); +}); + +test('moversOf lists price changes only, biggest drop first', () => { + const cmp = compareRuns( + [page(1, { B000000001: { p: 950 }, B000000002: { p: 500 }, B000000003: { p: 1100 }, B000000004: { p: 700 } })], + [page(1, { B000000001: { p: 1000 }, B000000002: { p: 1000 }, B000000003: { p: 1000 }, B000000004: { p: 700 } })], + ); + assert.deepEqual(moversOf(cmp.values()).map((c) => c.asin), ['B000000002', 'B000000001', 'B000000003']); +}); + +test('daysBetween counts whole days between run starts', () => { + const at = (ms) => ({ startedAt: { toMillis: () => ms } }); + assert.equal(daysBetween(at(8.5 * 86400000), at(1 * 86400000)), 7); +}); + +test('asSeenBy shows the run point and delta over the product document', () => { + const product = { asin: 'B000000001', mk: 'US', name: 'Mug', latest: { p: 1, runId: 'other' }, delta: { p: 5 } }; + const cmp = { asin: 'B000000001', now: { p: 900 }, prev: { p: 1000 }, delta: { p: -100, pPct: -10 } }; + const row = asSeenBy(product, cmp, { runId: 'k_mug_2', dayKey: '2026-06-09', startedAt: 't' }); + assert.equal(row.name, 'Mug'); + assert.deepEqual(row.latest, { p: 900, at: 't', runId: 'k_mug_2', dayKey: '2026-06-09' }); + assert.deepEqual(row.delta, { p: -100, pPct: -10 }); +}); diff --git a/firestore.indexes.json b/firestore.indexes.json index 3574969..9c7f4b4 100644 --- a/firestore.indexes.json +++ b/firestore.indexes.json @@ -4,20 +4,8 @@ { "fieldPath": "sourceId", "order": "ASCENDING" }, { "fieldPath": "startedAt", "order": "DESCENDING" } ] }, - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "lead.stage", "order": "ASCENDING" }, - { "fieldPath": "lead.stageChangedAt", "order": "DESCENDING" } ] }, - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ { "fieldPath": "sourceIds", "arrayConfig": "CONTAINS" }, - { "fieldPath": "latest.at", "order": "DESCENDING" } ] }, - - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "sourceIds", "arrayConfig": "CONTAINS" }, - { "fieldPath": "delta.pPct", "order": "ASCENDING" } ] }, - - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "tags", "arrayConfig": "CONTAINS" }, { "fieldPath": "latest.at", "order": "DESCENDING" } ] } ], "fieldOverrides": [ diff --git a/firestore.rules b/firestore.rules index a120a80..bbeee8c 100644 --- a/firestore.rules +++ b/firestore.rules @@ -1,4 +1,9 @@ rules_version = '2'; + +// Checked against the shared schema (packages/schema/index.js, SV 1). +// The extension writes sources, runs, pages, products and history; the +// dashboard writes users, workspaces, leads, tags, events and the watchlist +// fields on sources. Every document lists the keys it may carry. service cloud.firestore { match /databases/{db}/documents { function signedIn() { return request.auth != null; } @@ -7,77 +12,260 @@ service cloud.firestore { // Later (VA Seats): || request.auth.uid in get(/databases/$(db)/documents/workspaces/$(wid)).data.members } + function nat(v) { return v is int && v >= 0; } + function str(v, max) { return v is string && v.size() <= max; } + function strOrNull(v, max) { return v == null || str(v, max); } + function asinOk(a) { return a is string && a.matches('^[A-Z0-9]{10}$'); } + function dayKeyOk(k) { return k is string && k.matches('^[0-9]{4}-[0-9]{2}-[0-9]{2}$'); } + function sourceIdOk(id) { return id is string && id.matches('^[sk]_[A-Za-z0-9_-]{1,200}$'); } + + // Scan times come from the scanning machine's clock. An hour of slack + // lets a fast clock sync while still refusing a 2099 pin to the top. + function past(t) { return t is timestamp && t <= request.time + duration.value(1, 'h'); } + + // Set when the document is created, never changed after. + function createOnly(field) { + return resource == null + || request.resource.data.get(field, null) == resource.data.get(field, null); + } + + // A compact point (schema Point). `extra` names the provenance keys a + // point may also carry where it is stored. + function pointOk(pt, extra) { + return pt is map + && pt.keys().hasOnly(['p', 'r', 'v', 'pr', 'rk'].concat(extra)) + && (!('p' in pt) || nat(pt.p)) + && (!('r' in pt) || (pt.r is number && pt.r >= 0 && pt.r <= 5)) + && (!('v' in pt) || nat(pt.v)) + && (!('pr' in pt) || pt.pr in [0, 1]) + && (!('rk' in pt) || (pt.rk is int && pt.rk > 0)); + } + + function deltaOk(d) { + return d is map + && d.keys().hasOnly(['p', 'pPct', 'r', 'v', 'days']) + && (!('p' in d) || d.p is int) + && (!('pPct' in d) || d.pPct is number) + && (!('r' in d) || d.r is number) + && (!('v' in d) || d.v is int) + && (!('days' in d) || nat(d.days)); + } + match /users/{uid} { - allow read, write: if signedIn() && request.auth.uid == uid; + allow read, delete: if signedIn() && request.auth.uid == uid; + allow create, update: if signedIn() && request.auth.uid == uid + && request.resource.data.keys().hasOnly(['displayName', 'email', 'defaultWorkspace', 'createdAt']) + && strOrNull(request.resource.data.get('displayName', null), 200) + && strOrNull(request.resource.data.get('email', null), 320) + && request.resource.data.get('defaultWorkspace', uid) == uid + && (resource != null || request.resource.data.get('createdAt', request.time) == request.time) + && createOnly('createdAt'); } match /workspaces/{wid} { - allow read: if member(wid); - allow create: if member(wid) - && request.resource.data.ownerUid == request.auth.uid - && request.resource.data.get('plan', 'free') == 'free'; - allow update: if member(wid) - && request.resource.data.ownerUid == resource.data.ownerUid - && request.resource.data.get('plan', 'free') == resource.data.get('plan', 'free'); - allow delete: if false; + function workspaceOk() { + let d = request.resource.data; + return d.keys().hasOnly(['ownerUid', 'name', 'plan', 'settings', 'tagMeta', 'lastReviewedAt', 'createdAt']) + && d.ownerUid == wid + && strOrNull(d.get('name', null), 120) + && d.get('settings', {}) is map + && d.get('tagMeta', {}) is map + && (!('lastReviewedAt' in d) || past(d.lastReviewedAt)); + } + + allow read, delete: if member(wid); + allow create: if member(wid) && workspaceOk() + && request.resource.data.get('plan', 'free') == 'free' + && request.resource.data.get('createdAt', request.time) == request.time; + allow update: if member(wid) && workspaceOk() + && request.resource.data.get('plan', null) == resource.data.get('plan', null) + && createOnly('createdAt'); match /sources/{sourceId} { + function sourceOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'sourceId', 'type', 'sellerId', 'keyword', 'url', 'lastRunId', 'lastScrapedAt', 'catalogSize', + 'nickname', 'watched', 'cadenceDays', 'tags', 'notes', 'createdAt']) + && sourceIdOk(sourceId) + && d.get('sv', 1) == 1 + && d.get('sourceId', sourceId) == sourceId + && d.type in ['storefront', 'keyword'] + && strOrNull(d.get('sellerId', null), 40) + && strOrNull(d.get('keyword', null), 300) + && strOrNull(d.get('url', null), 2000) + && (!('lastRunId' in d) || str(d.lastRunId, 260)) + && (!('lastScrapedAt' in d) || past(d.lastScrapedAt)) + && (!('catalogSize' in d) || nat(d.catalogSize)) + && strOrNull(d.get('nickname', null), 120) + && d.get('watched', false) is bool + && (!('cadenceDays' in d) || (d.cadenceDays is int && d.cadenceDays >= 1 && d.cadenceDays <= 365)) + && (!('tags' in d) || (d.tags is list && d.tags.size() <= 30)) + && strOrNull(d.get('notes', null), 5000) + && createOnly('createdAt'); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.type in ['storefront', 'keyword'] - && request.resource.data.get('cadenceDays', 7) is int; + allow create, update: if member(wid) && sourceOk(); } match /runs/{runId} { + function runOk() { + let d = request.resource.data; + let c = d.counters; + return d.keys().hasOnly([ + 'sv', 'runId', 'sourceId', 'source', 'mk', 'dayKey', 'startedAt', 'finishedAt', 'status', 'reason', + 'pagesDone', 'maxPages', 'pagesPlanned', 'totalResultsOnSerp', 'counters', 'label']) + && d.sv == 1 + && sourceIdOk(d.sourceId) + && d.runId == runId + && d.startedAt is timestamp + && runId == d.sourceId + '_' + string(d.startedAt.toMillis()) + && past(d.startedAt) + && d.source is map + && d.source.keys().hasOnly(['type', 'sellerId', 'keyword', 'url']) + && d.source.type in ['storefront', 'keyword'] + && d.mk == 'US' + && dayKeyOk(d.dayKey) + && d.status in ['active', 'complete', 'stopped', 'dead'] + && (d.finishedAt == null || (past(d.finishedAt) && d.finishedAt >= d.startedAt)) + && (d.status != 'active' || d.finishedAt == null) + && strOrNull(d.reason, 100) + && nat(d.pagesDone) + && d.maxPages is int && d.maxPages >= 1 && d.maxPages <= 1000 + && (d.pagesPlanned == null || nat(d.pagesPlanned)) + && (d.totalResultsOnSerp == null || nat(d.totalResultsOnSerp)) + && c is map + && c.keys().hasOnly(['placements', 'uniqueAsins', 'sponsored', 'priceParseFailures', 'newSeen']) + && nat(c.placements) && nat(c.uniqueAsins) && nat(c.sponsored) + && nat(c.priceParseFailures) && nat(c.newSeen) + && strOrNull(d.get('label', null), 120); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.status in ['active', 'complete', 'stopped', 'dead'] - && request.resource.data.dayKey is string - && request.resource.data.sourceId is string; + allow create, update: if member(wid) && runOk(); match /pages/{pageId} { + function pageOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'runId', 'page', 'scrapedAt', 'expireAt', 'count', 'placements', 'kind', 'items', 'total', 'truncated']) + && d.sv == 1 + && d.runId == runId + && d.page is int && d.page >= 1 && d.page <= 1000 + && pageId.matches('^p[0-9]{4,}$') + && int(pageId[1:pageId.size()]) == d.page + && past(d.scrapedAt) + // TTL field: 400 days after the run started, so never further out than that. + && d.expireAt is timestamp + && d.expireAt <= request.time + duration.value(401, 'd') + && nat(d.count) + && nat(d.placements) + && str(d.kind, 20) + // Items can't be looped over here; the extension validates each. + && d.items is map + && d.items.size() <= 120 + && (!('total' in d) || nat(d.total)) + && (!('truncated' in d) || d.truncated is bool); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.items is map - && request.resource.data.items.size() <= 120 // Amazon pages carry ≤60 placements; 2× headroom - && request.resource.data.expireAt is timestamp; // TTL field is mandatory on cold docs + allow create, update: if member(wid) && pageOk(); } } match /products/{asin} { + function leadOk(lead) { + return lead is map + && lead.keys().hasOnly([ + 'stage', 'stageChangedAt', 'rejectedReason', 'notes', 'buyPrice', 'qty', 'supplier', 'orderRef', 'recheckAt']) + // A lead with no stored stage is 'new' (the drawer never writes it). + && lead.get('stage', 'new') in ['new', 'reviewing', 'approved', 'purchased', 'rejected', 'archived'] + // A stage move is stamped by the server, not the browser. + && (!('stageChangedAt' in lead) + || lead.stageChangedAt == request.time + || (resource != null + && resource.data.get('lead', {}).get('stageChangedAt', null) == lead.stageChangedAt)) + && strOrNull(lead.get('rejectedReason', null), 500) + && strOrNull(lead.get('notes', null), 5000) + && (lead.get('buyPrice', null) == null || nat(lead.buyPrice)) + && (lead.get('qty', null) == null || nat(lead.qty)) + && strOrNull(lead.get('supplier', null), 200) + && strOrNull(lead.get('orderRef', null), 200) + && (lead.get('recheckAt', null) == null || lead.recheckAt is timestamp); + } + + function productOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'asin', 'mk', 'name', 'url', 'img', 'latest', 'prev', 'delta', 'sourceIds', 'firstSeenAt', 'firstRunId', + 'lead', 'tags', 'verdict']) + && asinOk(asin) + && d.asin == asin // doc id == payload ASIN (dedup axis) + && d.get('sv', 1) == 1 + && d.get('mk', 'US') == 'US' + && strOrNull(d.get('name', null), 1000) + && (!('url' in d) || (str(d.url, 500) && d.url.matches('^https://www[.]amazon[.]com/.*'))) + && (d.get('img', null) == null || (str(d.img, 1000) && d.img.matches('^https://.*'))) + && (!('latest' in d) || ( + pointOk(d.latest, ['at', 'runId', 'dayKey']) + && past(d.latest.at) + && str(d.latest.runId, 260) + && dayKeyOk(d.latest.dayKey))) + && (d.get('prev', null) == null || ( + pointOk(d.prev, ['at']) + && (!('at' in d.prev) || past(d.prev.at)))) + && (d.get('delta', null) == null || deltaOk(d.delta)) + && (!('sourceIds' in d) || (d.sourceIds is list && d.sourceIds.size() <= 200)) + && (!('firstSeenAt' in d) || past(d.firstSeenAt)) + && (!('firstRunId' in d) || str(d.firstRunId, 260)) + && createOnly('firstSeenAt') + && createOnly('firstRunId') + && (!('lead' in d) || leadOk(d.lead)) + && (!('tags' in d) || (d.tags is list && d.tags.size() <= 30)) + && (!('verdict' in d) || (d.verdict is map && d.verdict.size() <= 8)); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.asin == asin // doc id == payload ASIN (dedup axis) - && (!('latest' in request.resource.data) || ( - request.resource.data.latest.dayKey is string - && (!('p' in request.resource.data.latest) || request.resource.data.latest.p is int) - && (!('v' in request.resource.data.latest) || request.resource.data.latest.v is int))) - // A lead with no stored stage is 'new' (the drawer never writes it). - && (!('lead' in request.resource.data) - || (request.resource.data.lead is map - && request.resource.data.lead.get('stage', 'new') in - ['new', 'reviewing', 'approved', 'purchased', 'rejected', 'archived'])); + allow create, update: if member(wid) && productOk(); match /history/{seriesId} { - allow read: if member(wid); - allow write: if member(wid) && seriesId in ['daily', 'monthly']; + allow read, delete: if member(wid); + allow create, update: if member(wid) + && seriesId in ['daily', 'monthly'] + && request.resource.data.keys().hasOnly(['sv', 'asin', 'd']) + && request.resource.data.sv == 1 + && request.resource.data.asin == asin + && request.resource.data.d is map + && request.resource.data.d.size() <= 1500; } match /offerSnapshots/{snapId} { allow read, delete: if member(wid); allow create, update: if member(wid) && request.resource.data.prices is list - && request.resource.data.expireAt is timestamp; + && request.resource.data.prices.size() <= 200 + && request.resource.data.expireAt is timestamp + && request.resource.data.expireAt <= request.time + duration.value(401, 'd'); } match /events/{eventId} { - allow read, create: if member(wid); - allow update, delete: if false; // append-only activity trail + allow read, delete: if member(wid); + // Append-only activity trail, stamped by the server. + allow create: if member(wid) + && request.resource.data.keys().hasOnly(['type', 'from', 'to', 'at', 'note']) + && str(request.resource.data.type, 40) + && strOrNull(request.resource.data.get('from', null), 40) + && strOrNull(request.resource.data.get('to', null), 40) + && strOrNull(request.resource.data.get('note', null), 2000) + && request.resource.data.at == request.time; + allow update: if false; } } - match /sellers/{sellerId} { allow read, write: if member(wid); } - match /rules/{ruleSetId} { allow read, write: if member(wid); } - match /views/{viewId} { allow read, write: if member(wid); } - match /alerts/{alertId} { allow read, write: if member(wid); } + // Nothing writes these yet. Owners can read and clear them. + match /sellers/{sellerId} { allow read, delete: if member(wid); } + match /rules/{ruleSetId} { allow read, delete: if member(wid); } + match /views/{viewId} { allow read, delete: if member(wid); } + match /alerts/{alertId} { allow read, delete: if member(wid); } } } } diff --git a/package.json b/package.json index 1a3e3a7..d2632aa 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,15 @@ "deploy": "node scripts/deploy-check.mjs && npm run build && node scripts/check-bundle.mjs && firebase deploy --project proscanbot --only hosting,firestore:rules,firestore:indexes", "emulators": "node scripts/emulators.mjs start", "emulators:persist": "node scripts/emulators.mjs start --import=./.emulator-data --export-on-exit=./.emulator-data", - "seed": "node scripts/seed-emulators.mjs", - "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs\"", + "seed": "node scripts/seed-extension.mjs", + "seed:fixture": "node scripts/seed-emulators.mjs", + "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", + "test:e2e": "node scripts/emulators.mjs exec \"node scripts/e2e.mjs\"", "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", - "test": "npm run typecheck && npm run build && node scripts/check-bundle.mjs && npm run test:rules" + "test:unit": "node --experimental-strip-types --no-warnings --test \"dashboard/test/*.test.mjs\"", + "check:schema": "node scripts/check-schema.mjs", + "test": "npm run typecheck && npm run check:schema && npm run test:unit && npm run build && node scripts/check-bundle.mjs && npm run test:rules" }, "dependencies": { "@fontsource-variable/archivo": "^5.2.8", diff --git a/packages/schema/index.d.ts b/packages/schema/index.d.ts new file mode 100644 index 0000000..7b34172 --- /dev/null +++ b/packages/schema/index.d.ts @@ -0,0 +1,150 @@ +// Types for index.js, the cloud schema vendored from the extension +// (packages/schema/index.js there). index.js is copied byte for byte; +// this file is ours. scripts/check-schema.mjs keeps the two in step. + +export declare const SV: 1; +export declare const MK: 'US'; +export declare const ASIN_RE: RegExp; +export declare const DAY_KEY_RE: RegExp; +export declare const PAGE_TTL_DAYS: number; +export declare const MAX_PAGE_ITEMS: number; +export declare const RUN_STATUS: readonly ['active', 'complete', 'stopped', 'dead']; +export declare const SOURCE_TYPES: readonly ['storefront', 'keyword']; + +export type RunStatus = (typeof RUN_STATUS)[number]; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** Anything timeMs understands: ms, a Date, or a Timestamp. */ +export type TimeLike = number | Date | { toMillis(): number }; + +/** One observation. Unknown values are left out, never null or 0. */ +export interface Point { + /** price, cents */ + p?: number; + /** rating, 0 to 5 */ + r?: number; + /** review count */ + v?: number; + pr?: 0 | 1; + /** organic rank within the run, from 1 */ + rk?: number; + /** sponsored card, page items only */ + sp?: 0 | 1; +} + +export interface Delta { + p?: number; + pPct?: number; + r?: number; + v?: number; + days?: number; +} + +export interface Source { + type: SourceType; + sellerId: string | null; + keyword: string | null; + url: string | null; +} + +export interface SourceDoc { + sv: number; + sourceId: string; + type: SourceType; + sellerId: string | null; + keyword: string | null; + url: string | null; + lastRunId: string; + lastScrapedAt: T; + catalogSize?: number; +} + +export interface RunCounters { + placements: number; + uniqueAsins: number; + sponsored: number; + priceParseFailures: number; + newSeen: number; +} + +export interface RunDoc { + sv: number; + runId: string; + sourceId: string; + source: Source; + mk: string; + dayKey: string; + startedAt: T; + finishedAt: T | null; + status: RunStatus; + reason: string | null; + pagesDone: number; + maxPages: number; + pagesPlanned: number | null; + totalResultsOnSerp: number | null; + counters: RunCounters; +} + +export interface PageDoc { + sv: number; + runId: string; + page: number; + scrapedAt: T; + expireAt: T; + count: number; + placements: number; + kind: string; + items: Record; + /** result count Amazon showed; sync-plan.js writes it, the typedef there omits it */ + total?: number; + truncated?: boolean; +} + +export interface ProductDoc { + sv: number; + asin: string; + mk: string; + name?: string | null; + url: string; + img?: string | null; + latest: Point & { at: T; runId: string; dayKey: string }; + prev: (Point & { at?: T }) | null; + delta: Delta | null; + sourceIds: string[]; + firstSeenAt?: T; + firstRunId?: string; +} + +export interface HistoryDoc { + sv: number; + asin: string; + d: Record; +} + +export declare function hash(s: string): string; +export declare function slugify(s: string): string; +export declare function sourceOf(url: string): Source; +export declare function sourceIdOf(source: Source): string; +export declare function runIdOf(sourceId: string, startMs: number): string; +export declare function pageIdOf(page: number): string; +export declare function dayKeyOf(ms: number, tzOffsetMin?: number): string; +export declare function expireAtMs(startMs: number): number; +export declare function runStatusOf(state: string): RunStatus; +export declare function timeMs(v: unknown): number | null; +export declare function pointOf(rec: { + priceCents?: number | null; + rating?: number | null; + reviewCount?: number | null; + isPrime?: boolean; + organicRank?: number | null; +}): Point; +export declare function deltaOf(now: Point, prev: Point | null, days?: number | null): Delta | null; + +export declare function validateSource(doc: unknown): string[]; +export declare function validateRun(doc: unknown): string[]; +export declare function validatePage(doc: unknown): string[]; +export declare function validateProduct(doc: unknown): string[]; +export declare function validateHistory(doc: unknown): string[]; + +export type DocKind = 'source' | 'run' | 'page' | 'product' | 'history'; +export declare function assertValid(kind: DocKind, doc: D): D; diff --git a/packages/schema/index.js b/packages/schema/index.js new file mode 100644 index 0000000..edb605e --- /dev/null +++ b/packages/schema/index.js @@ -0,0 +1,480 @@ +/** + * @fileoverview The ProScan cloud schema: types, validators and id builders. + * + * One file shared by the extension (which writes) and the dashboard (which + * reads). It has no imports and no Firebase code, so the dashboard can copy + * it as is; keep the copies byte-identical and bump SV on any change to a + * document's shape. + * + * Conventions: + * - Money is integer cents. + * - Unknown is null, never 0. In compact points (latest, prev, history + * days, page items) an unknown value is left out, so read it as + * `point.p ?? null`. The Firestore rules check `p is int` when present. + * - Every document carries `sv`, the schema version it was written with. + * - Times are Firestore Timestamps in the cloud. Builders and validators + * here take anything `timeMs` understands: a number of ms, a Date, or an + * object with toMillis(). + * + * Paths, all under workspaces/{uid}: + * sources/{sourceId} SourceDoc + * runs/{runId} RunDoc + * runs/{runId}/pages/{pageId} PageDoc + * products/{asin} ProductDoc (extension fields; lead, + * tags and verdict are the dashboard's) + * products/{asin}/history/daily HistoryDoc + * + * @module Schema + */ + +/** Schema version written into every document. */ +export const SV = 1; + +/** The one marketplace so far. */ +export const MK = 'US'; + +export const ASIN_RE = /^[A-Z0-9]{10}$/; +export const DAY_KEY_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Page chunks expire this long after the run started (TTL policy on expireAt). */ +export const PAGE_TTL_DAYS = 400; + +/** The rules cap a page chunk's items map at this size. */ +export const MAX_PAGE_ITEMS = 120; + +export const RUN_STATUS = ['active', 'complete', 'stopped', 'dead']; +export const SOURCE_TYPES = ['storefront', 'keyword']; + +/** + * @typedef {Object} Point one observation of a product + * @property {number} [p] price, cents + * @property {number} [r] rating, 0 to 5, one decimal + * @property {number} [v] review count + * @property {0|1} [pr] Prime badge + * @property {number} [rk] organic rank within the run, from 1 + * @property {0|1} [sp] sponsored card (page items only) + */ + +/** + * @typedef {Object} Delta change against the previous observation + * @property {number} [p] price change, cents + * @property {number} [pPct] price change, percent of the previous price + * @property {number} [r] rating change + * @property {number} [v] review count change + * @property {number} [days] days between the two observations + */ + +/** + * @typedef {Object} Source + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + */ + +/** + * @typedef {Object} SourceDoc + * @property {number} sv + * @property {string} sourceId + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + * @property {string} lastRunId + * @property {*} lastScrapedAt + * @property {number} [catalogSize] result count Amazon showed, when known + */ + +/** + * @typedef {Object} RunDoc + * @property {number} sv + * @property {string} runId + * @property {string} sourceId + * @property {Source} source + * @property {string} mk + * @property {string} dayKey local date the run started, YYYY-MM-DD + * @property {*} startedAt + * @property {*} finishedAt null while active + * @property {'active'|'complete'|'stopped'|'dead'} status + * @property {?string} reason how it ended, in the extension's words + * @property {number} pagesDone + * @property {number} maxPages + * @property {?number} pagesPlanned known only once a run completes + * @property {?number} totalResultsOnSerp + * @property {{placements:number, uniqueAsins:number, sponsored:number, priceParseFailures:number, newSeen:number}} counters + */ + +/** + * @typedef {Object} PageDoc + * @property {number} sv + * @property {string} runId + * @property {number} page + * @property {*} scrapedAt + * @property {*} expireAt + * @property {number} count products first seen on this page + * @property {number} placements cards on this page + * @property {string} kind results or last + * @property {Object} items every ASIN on the page + * @property {boolean} [truncated] more than MAX_PAGE_ITEMS ASINs + */ + +/** + * @typedef {Object} ProductDoc + * @property {number} sv + * @property {string} asin + * @property {string} mk + * @property {?string} name + * @property {string} url + * @property {?string} img + * @property {Point & {at:*, runId:string, dayKey:string}} latest + * @property {?(Point & {at:*})} prev + * @property {?Delta} delta + * @property {string[]} sourceIds + * @property {*} [firstSeenAt] set once, when the document is created + * @property {string} [firstRunId] + */ + +/** + * @typedef {Object} HistoryDoc + * @property {number} sv + * @property {string} asin + * @property {Object} d dayKey to point + */ + +// ── Ids ───────────────────────────────────────────────────────── + +/** 32-bit FNV-1a of a string, base 36. */ +export function hash(s) { + let h = 0x811c9dc5; + for (const ch of String(s)) { + const c = ch.codePointAt(0); + h ^= c & 0xff; + h = Math.imul(h, 0x01000193); + if (c > 0xff) { + h ^= c >>> 8; + h = Math.imul(h, 0x01000193); + } + } + return (h >>> 0).toString(36); +} + +export function slugify(s) { + return String(s).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); +} + +const SELLER_RE = /^[A-Z0-9]{6,20}$/; +// Query params that change per visit, not per search. +const NOISE = new Set(['page', 'ref', 'qid', 'xpid', 'crid', 'sprefix', 'dib', 'dib_tag', 'sr', 'ds', 'pd_rd_r', 'pf_rd_r']); + +/** The seller in a search URL: me=, seller=, or p_6:/me: inside rh=. */ +function sellerIn(params) { + const direct = params.get('me') || params.get('seller'); + if (direct) return direct.toUpperCase(); + const m = (params.get('rh') || '').match(/(?:^|,)(?:p_6|me):([A-Za-z0-9]+)/); + return m ? m[1].toUpperCase() : null; +} + +/** + * What an Amazon search URL scans. + * @param {string} url + * @returns {Source} + */ +export function sourceOf(url) { + let params = null; + try { + params = new URL(url).searchParams; + } catch (e) { + return { type: 'keyword', sellerId: null, keyword: null, url: url || null }; + } + const seller = sellerIn(params); + if (seller && SELLER_RE.test(seller)) { + return { type: 'storefront', sellerId: seller, keyword: params.get('k') || null, url }; + } + const keyword = params.get('k') || params.get('field-keywords') || null; + return { type: 'keyword', sellerId: null, keyword: keyword ? keyword.trim() : null, url }; +} + +/** + * The source id: s_{sellerId} for a storefront, k_{slug} for a keyword. + * A keyword with non-ASCII letters, or one too long, keeps what it can of + * the slug and adds a hash, so two keywords never share an id. A search + * with neither gets k_x{hash} of its query. + * @param {Source} source + * @returns {string} + */ +export function sourceIdOf(source) { + if (source && source.type === 'storefront' && source.sellerId) return `s_${source.sellerId}`; + const kw = source && source.keyword ? String(source.keyword).normalize('NFC').trim().toLowerCase() : ''; + if (kw) { + const slug = slugify(kw); + // eslint-disable-next-line no-control-regex + const plain = !/[^\x00-\x7f]/.test(kw) && slug.length > 0 && slug.length <= 80; + return plain ? `k_${slug}` : `k_${slug ? slug.slice(0, 60) + '-' : ''}h${hash(kw)}`; + } + let query = ''; + try { + const u = new URL(source.url); + query = [...u.searchParams.entries()] + .filter(([k]) => !NOISE.has(k)) + .map(([k, v]) => `${k}=${v}`) + .sort() + .join('&'); + query = u.pathname + '?' + query; + } catch (e) { /* no URL */ } + return query ? `k_x${hash(query)}` : 'k_unknown'; +} + +/** Run id: {sourceId}_{startMs}, minted once when the run starts. */ +export function runIdOf(sourceId, startMs) { + return `${sourceId}_${startMs}`; +} + +/** Page chunk id: p0001 for page 1. */ +export function pageIdOf(page) { + return 'p' + String(page).padStart(4, '0'); +} + +/** + * The local date at `ms` as YYYY-MM-DD. `tzOffsetMin` is what + * Date#getTimezoneOffset gives where the scan ran (minutes behind UTC), so + * an evening scan in the US files under its own day. + */ +export function dayKeyOf(ms, tzOffsetMin = 0) { + return new Date(ms - tzOffsetMin * 60000).toISOString().slice(0, 10); +} + +export function expireAtMs(startMs) { + return startMs + PAGE_TTL_DAYS * 86400000; +} + +/** + * The cloud status for an extension run: active while it runs, then + * complete, stopped (by the user or a block page) or dead. + */ +export function runStatusOf(state) { + switch (state) { + case 'starting': + case 'running': + case 'stopping': + return 'active'; + case 'done': + return 'complete'; + case 'stopped': + case 'blocked': + return 'stopped'; + default: + return 'dead'; + } +} + +// ── Values ────────────────────────────────────────────────────── + +/** Milliseconds from a number, Date, ISO string or Timestamp; null otherwise. */ +export function timeMs(v) { + if (typeof v === 'number') return Number.isFinite(v) ? v : null; + if (v instanceof Date) return Number.isNaN(v.getTime()) ? null : v.getTime(); + if (typeof v === 'string') { + const t = Date.parse(v); + return Number.isNaN(t) ? null : t; + } + if (v && typeof v.toMillis === 'function') return v.toMillis(); + return null; +} + +const isInt = (v) => Number.isInteger(v); +const round1 = (n) => Math.round(n * 10) / 10; + +/** + * A compact point from an extension product record, leaving out what is + * unknown. + * @param {{priceCents?:?number, rating?:?number, reviewCount?:?number, isPrime?:boolean, organicRank?:?number}} rec + * @returns {Point} + */ +export function pointOf(rec) { + const pt = {}; + if (isInt(rec.priceCents) && rec.priceCents >= 0) pt.p = rec.priceCents; + if (typeof rec.rating === 'number' && rec.rating > 0 && rec.rating <= 5) pt.r = round1(rec.rating); + if (isInt(rec.reviewCount) && rec.reviewCount >= 0) pt.v = rec.reviewCount; + if (typeof rec.isPrime === 'boolean') pt.pr = rec.isPrime ? 1 : 0; + if (isInt(rec.organicRank) && rec.organicRank > 0) pt.rk = rec.organicRank; + return pt; +} + +/** + * The delta between two points, or null when nothing can be compared. + * @param {Point} now + * @param {?Point} prev + * @param {?number} [days] + * @returns {?Delta} + */ +export function deltaOf(now, prev, days = null) { + if (!prev) return null; + const d = {}; + if (isInt(now.p) && isInt(prev.p)) { + d.p = now.p - prev.p; + if (prev.p > 0) d.pPct = round1((d.p / prev.p) * 100); + } + if (typeof now.r === 'number' && typeof prev.r === 'number') d.r = round1(now.r - prev.r); + if (isInt(now.v) && isInt(prev.v)) d.v = now.v - prev.v; + if (Object.keys(d).length === 0) return null; + if (isInt(days) && days >= 0) d.days = days; + return d; +} + +// ── Validators ────────────────────────────────────────────────── +// Each returns a list of problems; empty means valid. + +const isStr = (v) => typeof v === 'string'; +const isStrOrNull = (v) => v === null || isStr(v); +const isTime = (v) => timeMs(v) !== null && typeof v !== 'string'; + +function checkPoint(pt, where, errs, { sp = false } = {}) { + if (!pt || typeof pt !== 'object') { errs.push(`${where}: not a map`); return; } + for (const [k, v] of Object.entries(pt)) { + if (v === null || v === undefined) errs.push(`${where}.${k}: unknown values are left out, not null`); + } + if ('p' in pt && !(isInt(pt.p) && pt.p >= 0)) errs.push(`${where}.p: cents must be a whole number`); + if ('r' in pt && !(typeof pt.r === 'number' && pt.r >= 0 && pt.r <= 5)) errs.push(`${where}.r: rating out of range`); + if ('v' in pt && !(isInt(pt.v) && pt.v >= 0)) errs.push(`${where}.v: review count must be a whole number`); + if ('pr' in pt && pt.pr !== 0 && pt.pr !== 1) errs.push(`${where}.pr: must be 0 or 1`); + if ('rk' in pt && !(isInt(pt.rk) && pt.rk > 0)) errs.push(`${where}.rk: rank must be 1 or more`); + if ('sp' in pt && (!sp || (pt.sp !== 0 && pt.sp !== 1))) errs.push(`${where}.sp: only page items carry sp, as 0 or 1`); +} + +function checkSv(doc, errs) { + if (!doc || typeof doc !== 'object') { errs.push('not a document'); return false; } + if (doc.sv !== SV) errs.push(`sv: expected ${SV}`); + return true; +} + +/** @returns {string[]} */ +export function validateSource(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.sourceId) || !/^[sk]_/.test(doc.sourceId)) errs.push('sourceId: expected s_ or k_'); + if (!SOURCE_TYPES.includes(doc.type)) errs.push('type: storefront or keyword'); + if (!isStrOrNull(doc.sellerId)) errs.push('sellerId: string or null'); + if (!isStrOrNull(doc.keyword)) errs.push('keyword: string or null'); + if (!isStrOrNull(doc.url)) errs.push('url: string or null'); + if (!isStr(doc.lastRunId)) errs.push('lastRunId: string'); + if (!isTime(doc.lastScrapedAt)) errs.push('lastScrapedAt: time'); + if ('catalogSize' in doc && !(isInt(doc.catalogSize) && doc.catalogSize >= 0)) errs.push('catalogSize: whole number'); + return errs; +} + +/** @returns {string[]} */ +export function validateRun(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isStr(doc.sourceId)) errs.push('sourceId: string'); + if (isStr(doc.runId) && isStr(doc.sourceId) && !doc.runId.startsWith(doc.sourceId + '_')) errs.push('runId: must start with sourceId_'); + if (!doc.source || !SOURCE_TYPES.includes(doc.source.type)) errs.push('source.type: storefront or keyword'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if (!isStr(doc.dayKey) || !DAY_KEY_RE.test(doc.dayKey)) errs.push('dayKey: YYYY-MM-DD'); + if (!isTime(doc.startedAt)) errs.push('startedAt: time'); + if (doc.finishedAt !== null && !isTime(doc.finishedAt)) errs.push('finishedAt: time or null'); + if (!RUN_STATUS.includes(doc.status)) errs.push(`status: one of ${RUN_STATUS.join(', ')}`); + if (doc.status === 'active' && doc.finishedAt !== null) errs.push('finishedAt: null while active'); + if (!isStrOrNull(doc.reason)) errs.push('reason: string or null'); + if (!isInt(doc.pagesDone) || doc.pagesDone < 0) errs.push('pagesDone: whole number'); + if (!isInt(doc.maxPages) || doc.maxPages < 1) errs.push('maxPages: whole number'); + if (doc.pagesPlanned !== null && !isInt(doc.pagesPlanned)) errs.push('pagesPlanned: whole number or null'); + if (doc.totalResultsOnSerp !== null && !isInt(doc.totalResultsOnSerp)) errs.push('totalResultsOnSerp: whole number or null'); + const c = doc.counters || {}; + for (const k of ['placements', 'uniqueAsins', 'sponsored', 'priceParseFailures', 'newSeen']) { + if (!isInt(c[k]) || c[k] < 0) errs.push(`counters.${k}: whole number`); + } + return errs; +} + +/** @returns {string[]} */ +export function validatePage(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isInt(doc.page) || doc.page < 1) errs.push('page: 1 or more'); + if (!isTime(doc.scrapedAt)) errs.push('scrapedAt: time'); + if (!isTime(doc.expireAt)) errs.push('expireAt: time'); + if (!isInt(doc.count) || doc.count < 0) errs.push('count: whole number'); + if (!isInt(doc.placements) || doc.placements < 0) errs.push('placements: whole number'); + if (!doc.items || typeof doc.items !== 'object') { + errs.push('items: map'); + } else { + const keys = Object.keys(doc.items); + if (keys.length > MAX_PAGE_ITEMS) errs.push(`items: at most ${MAX_PAGE_ITEMS}`); + for (const asin of keys) { + if (!ASIN_RE.test(asin)) errs.push(`items.${asin}: not an ASIN`); + checkPoint(doc.items[asin], `items.${asin}`, errs, { sp: true }); + } + } + return errs; +} + +/** @returns {string[]} */ +export function validateProduct(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if ('name' in doc && !isStrOrNull(doc.name)) errs.push('name: string or null'); + if ('img' in doc && doc.img !== null && !(isStr(doc.img) && /^https:\/\//.test(doc.img))) errs.push('img: https URL or null'); + if (!isStr(doc.url)) errs.push('url: string'); + if (!doc.latest) { + errs.push('latest: map'); + } else { + const { at, runId, dayKey, ...pt } = doc.latest; + checkPoint(pt, 'latest', errs); + if (!isTime(at)) errs.push('latest.at: time'); + if (!isStr(runId)) errs.push('latest.runId: string'); + if (!isStr(dayKey) || !DAY_KEY_RE.test(dayKey)) errs.push('latest.dayKey: YYYY-MM-DD'); + } + if (doc.prev !== null && doc.prev !== undefined) { + const { at, ...pt } = doc.prev; + checkPoint(pt, 'prev', errs); + if (at !== undefined && !isTime(at)) errs.push('prev.at: time'); + } + if (doc.delta !== null && doc.delta !== undefined) { + const d = doc.delta; + if ('p' in d && !isInt(d.p)) errs.push('delta.p: cents must be a whole number'); + if ('pPct' in d && typeof d.pPct !== 'number') errs.push('delta.pPct: number'); + if ('r' in d && typeof d.r !== 'number') errs.push('delta.r: number'); + if ('v' in d && !isInt(d.v)) errs.push('delta.v: whole number'); + if ('days' in d && !isInt(d.days)) errs.push('delta.days: whole number'); + for (const [k, v] of Object.entries(d)) if (v === null) errs.push(`delta.${k}: left out, not null`); + } + if ('sourceIds' in doc && !(Array.isArray(doc.sourceIds) && doc.sourceIds.every(isStr))) errs.push('sourceIds: list of strings'); + if ('firstSeenAt' in doc && !isTime(doc.firstSeenAt)) errs.push('firstSeenAt: time'); + return errs; +} + +/** @returns {string[]} */ +export function validateHistory(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (!doc.d || typeof doc.d !== 'object') { + errs.push('d: map'); + } else { + for (const [day, pt] of Object.entries(doc.d)) { + if (!DAY_KEY_RE.test(day)) errs.push(`d.${day}: not a day key`); + checkPoint(pt, `d.${day}`, errs); + } + } + return errs; +} + +const VALIDATORS = { + source: validateSource, + run: validateRun, + page: validatePage, + product: validateProduct, + history: validateHistory +}; + +/** Throws when `doc` is not a valid document of `kind`. */ +export function assertValid(kind, doc) { + const errs = VALIDATORS[kind](doc); + if (errs.length) throw new Error(`invalid ${kind} document: ${errs.join('; ')}`); + return doc; +} diff --git a/scripts/check-schema.mjs b/scripts/check-schema.mjs new file mode 100644 index 0000000..8c35361 --- /dev/null +++ b/scripts/check-schema.mjs @@ -0,0 +1,38 @@ +// The vendored schema must be the extension's file byte for byte, and +// index.d.ts must declare everything index.js exports. +// +// node scripts/check-schema.mjs + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { WEB_ROOT, findExtension } from './lib/extension.mjs'; + +const ours = resolve(WEB_ROOT, 'packages', 'schema', 'index.js'); +const types = readFileSync(resolve(WEB_ROOT, 'packages', 'schema', 'index.d.ts'), 'utf8'); +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` (${detail})`}`); +}; + +// Line endings are ignored: autocrlf checks files out with CRLF on Windows. +const lf = (text) => text.replace(/\r\n/g, '\n'); +const src = lf(readFileSync(ours, 'utf8')); +const exported = [...src.matchAll(/^export (?:const|function) (\w+)/gm)].map((m) => m[1]); +const missing = exported.filter((name) => !new RegExp(`declare (?:const|function) ${name}\\b`).test(types)); +check('index.d.ts declares every export', missing.length === 0, missing.join(', ')); + +// Without the extension only the byte comparison is skipped; the check +// above still decides the exit code. +const ext = findExtension(); +if (ext) { + const theirs = lf(readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8')); + check(`index.js matches ${ext}`, theirs === src, 'copy the extension file over ours'); +} else if (process.env.PROSCAN_ALLOW_NO_EXT === '1') { + console.log(' SKIP index.js matches the extension: no extension checkout (PROSCAN_ALLOW_NO_EXT=1)'); +} else { + check('extension checkout found', false, 'set PROSCAN_EXT, or PROSCAN_ALLOW_NO_EXT=1 to skip'); +} + +console.log(failed === 0 ? 'RESULT: PASS' : `RESULT: FAIL (${failed})`); +process.exit(failed === 0 ? 0 : 1); diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs new file mode 100644 index 0000000..1cc47d6 --- /dev/null +++ b/scripts/e2e.mjs @@ -0,0 +1,436 @@ +// End to end: the extension's real engine and sync module (from its +// checkout) seed the emulator with 1,050 products over three runs on +// different days, then the production dashboard bundle, built against the +// emulators and served by `vite preview`, is signed in to and checked: +// every promised column fills, counts are true, Load more pages on, Movers +// compares runs, Export writes the whole scope, and a Google account can +// set a password the extension then signs in with (F-50). +// +// npm run test:e2e +// +// Runs inside `firebase emulators:exec` (scripts/emulators.mjs). No Amazon +// traffic: every amazon.com and media-amazon.com request is answered here. + +import { spawn, spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { chromium } from 'playwright'; +import * as XLSX from 'xlsx'; +import { initializeApp, deleteApp } from 'firebase/app'; +import { + getAuth, + connectAuthEmulator, + createUserWithEmailAndPassword, + GoogleAuthProvider, + signInWithCredential, + signInWithEmailAndPassword, +} from 'firebase/auth'; +import { getFirestore, connectFirestoreEmulator, terminate } from 'firebase/firestore'; +import { requireExtension, WEB_ROOT } from './lib/extension.mjs'; +import { asinOf, cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST; +const FS_HOST = process.env.FIRESTORE_EMULATOR_HOST; +if (!AUTH_HOST || !FS_HOST) throw new Error('run through npm run test:e2e, which starts the emulators'); + +const PREVIEW_PORT = Number(process.env.E2E_PREVIEW_PORT || 4179); +const BASE = `http://localhost:${PREVIEW_PORT}/dashboard/`; +const OUT_DIR = resolve(WEB_ROOT, 'dist-e2e', 'dashboard'); +const SHOTS = resolve(WEB_ROOT, '.screenshots', 'e2e'); +const DEADLINE = Date.now() + 7 * 60 * 1000; +const DAY = 86400000; + +const EMAIL = `e2e-${Date.now()}@proscan.test`; +const PASSWORD = 'e2e-pass-1234'; +const STORE = 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2&marketplaceID=ATVPDKIKX0DER'; +const KEYWORD = 'https://www.amazon.com/s?k=stainless+tumbler'; + +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` (${detail})` : ''}`); +}; +// Bugs that live in the extension: reported, not fixed here. Strict, so +// the mark has to come off once the extension fixes them. +let known = 0; +const knownFailure = (name, ok, detail = '') => { + if (ok) { + failed++; + console.log(` FAIL ${name} now passes; remove its known-failure mark`); + } else { + known++; + console.log(` KNOWN FAILURE ${name}${detail ? ` (${detail})` : ''}`); + } +}; +const fmt = (n) => n.toLocaleString('en-US'); + +function firebase(name) { + const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, name); + const auth = getAuth(app); + const db = getFirestore(app); + connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); + const [h, p] = FS_HOST.split(':'); + connectFirestoreEmulator(db, h, Number(p)); + return { app, auth, db }; +} + +/* ── the scan data ───────────────────────────────────────────────── */ + +// Run 2 moves a third of the storefront down, a third up, keeps the rest, +// and loses every 10th price to a parse failure. +const run2Price = (i) => (i % 10 === 0 ? null : i % 3 === 0 ? 1000 + i - 50 * ((i % 7) + 1) : i % 3 === 1 ? 1000 + i + 25 : 1000 + i); +const STORE_N = 700; +const KEYWORD_FROM = 650; +const KEYWORD_N = 400; +const UNIQUE = KEYWORD_FROM + KEYWORD_N; // 1,050 +let expectedMovers = 0; +for (let i = 0; i < STORE_N; i++) { + const p = run2Price(i); + if (p !== null && p !== 1000 + i) expectedMovers++; +} + +async function seed() { + const ext = await loadExtension(requireExtension('the e2e seed')); + const fb = firebase('e2e-seed'); + const uid = (await createUserWithEmailAndPassword(fb.auth, EMAIL, PASSWORD)).user.uid; + const install = installExtension(ext, { uid, db: fb.db }); + const day1 = Date.parse('2026-09-01T14:00:00Z'); + const runs = [ + await install.scrape(STORE, day1, cards(STORE_N), 48), + await install.scrape(STORE, day1 + 7 * DAY, cards(STORE_N, { price: run2Price }), 48), + await install.scrape(KEYWORD, day1 + 8 * DAY, cards(KEYWORD_N, { offset: KEYWORD_FROM }), 48), + ]; + const t0 = Date.now(); + const totals = await install.flush(); + console.log(`[seed] ${JSON.stringify(totals)} in ${Math.round((Date.now() - t0) / 1000)}s`); + check('the extension synced three runs', totals.runs === 3 && (await install.pending()) === 0, JSON.stringify(totals)); + await terminate(fb.db); + await deleteApp(fb.app); + return { uid, runs, day1, ext }; +} + +/* ── the production bundle ───────────────────────────────────────── */ + +function build() { + rmSync(resolve(WEB_ROOT, 'dist-e2e'), { recursive: true, force: true }); + const res = spawnSync('npx', ['vite', 'build', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--emptyOutDir'], { + cwd: WEB_ROOT, + shell: true, + stdio: 'inherit', + env: { ...process.env, VITE_USE_EMULATOR: 'true' }, + timeout: 3 * 60 * 1000, + }); + if (res.status !== 0) throw new Error('dashboard build failed'); +} + +async function preview() { + // A server left over from an earlier run would stand in for this build. + const taken = await fetch(BASE).then(() => true, () => false); + if (taken) throw new Error(`port ${PREVIEW_PORT} already answers; stop that server or set E2E_PREVIEW_PORT`); + const child = spawn( + 'npx', + ['vite', 'preview', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--port', String(PREVIEW_PORT), '--strictPort'], + // Its own process group off Windows, so kill() takes vite down with npx. + { cwd: WEB_ROOT, shell: true, stdio: 'ignore', detached: process.platform !== 'win32' }, + ); + for (let i = 0; i < 60; i++) { + if (child.exitCode !== null) throw new Error(`vite preview exited with ${child.exitCode}`); + try { + const res = await fetch(BASE); + if (res.ok) return child; + } catch { + /* not up yet */ + } + await new Promise((r) => setTimeout(r, 500)); + } + kill(child); + throw new Error(`vite preview did not answer on ${BASE}`); +} + +function kill(child) { + if (!child || child.exitCode !== null) return; + if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(child.pid), '/T', '/F'], { stdio: 'ignore' }); + else { + try { + process.kill(-child.pid, 'SIGTERM'); + } catch { + child.kill('SIGTERM'); + } + } +} + +/* ── the browser ─────────────────────────────────────────────────── */ + +const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64'); + +async function newPage(browser, consoleErrors) { + const ctx = await browser.newContext({ viewport: { width: 1440, height: 1400 }, acceptDownloads: true }); + await ctx.route(/^https?:\/\/([^/]*\.)?(amazon\.com|media-amazon\.com)\//, (route) => + route.fulfill({ status: 200, contentType: 'image/png', body: PNG }), + ); + // Hosting serves public/ at the root, which vite preview of the dashboard + // alone does not; without this the favicon 404s into the console check. + await ctx.route(`http://localhost:${PREVIEW_PORT}/icon128.png`, (route) => + route.fulfill({ status: 200, contentType: 'image/png', path: resolve(WEB_ROOT, 'public', 'icon128.png') }), + ); + const page = await ctx.newPage(); + page.on('console', (msg) => { + if (msg.type() === 'error') consoleErrors.push(msg.text()); + }); + page.on('pageerror', (err) => consoleErrors.push(String(err))); + return { ctx, page }; +} + +async function signIn(page, email, password) { + await page.goto(BASE); + await page.waitForSelector('#auth-email', { timeout: 30000 }); + await page.fill('#auth-email', email); + await page.fill('#auth-password', password); + await page.click('.auth__cta'); +} + +const subtitle = (page) => page.locator('.page-header__sub').innerText(); + +async function waitSubtitle(page, re, timeout = 30000) { + await page.waitForFunction( + (src) => new RegExp(src).test(document.querySelector('.page-header__sub')?.textContent ?? ''), + re.source, + { timeout }, + ); + return subtitle(page); +} + +/** The rendered board as {headers, rows: [{Header: text}], imgs}. */ +function readBoard(page) { + return page.evaluate(() => { + const headers = [...document.querySelectorAll('.board-table thead th')].map((th) => th.textContent.trim()); + const rows = [...document.querySelectorAll('.board-row')].map((tr) => { + const cells = [...tr.querySelectorAll('td')]; + const out = {}; + headers.forEach((h, i) => (out[h] = (cells[i]?.textContent ?? '').trim())); + out.__img = tr.querySelector('img.board-prod__img')?.getAttribute('src') ?? ''; + return out; + }); + return { headers, rows }; + }); +} + +const header = (headers, prefix) => headers.find((h) => h.startsWith(prefix)); +const share = (rows, fn) => (rows.length === 0 ? 0 : rows.filter(fn).length / rows.length); + +async function checkColumns(page, label, { deltas }) { + const { headers, rows } = await readBoard(page); + const need = ['Product', 'Price', 'Δ Price', 'Rating', 'Reviews', 'Rank']; + const missing = need.filter((h) => !header(headers, h)); + check(`${label}: promised columns are shown`, missing.length === 0, missing.length ? `missing ${missing.join(', ')}` : headers.join(' | ')); + check(`${label}: rows render`, rows.length >= 20, `${rows.length} rows`); + const col = (h) => header(headers, h); + check(`${label}: every row has a name`, share(rows, (r) => /Test product \d+/.test(r[col('Product')])) === 1); + check(`${label}: every row has an image`, share(rows, (r) => r.__img.startsWith('https://m.media-amazon.com/')) === 1); + const price = share(rows, (r) => /^\$\d/.test(r[col('Price')])); + check(`${label}: price fills`, price >= 0.85, `${Math.round(price * 100)}%`); + check(`${label}: rating fills`, share(rows, (r) => /^\d\.\d/.test(r[col('Rating')])) === 1); + check(`${label}: reviews fill`, share(rows, (r) => /^\d/.test(r[col('Reviews')])) === 1); + check(`${label}: rank fills`, share(rows, (r) => /^#\d+/.test(r[col('Rank')])) === 1); + if (deltas) { + const d = share(rows, (r) => /\$\d/.test(r[col('Δ Price')])); + check(`${label}: Δ price fills where the price moved`, d >= deltas, `${Math.round(d * 100)}%`); + } + return { headers, rows }; +} + +async function main() { + mkdirSync(SHOTS, { recursive: true }); + console.log('[seed] the extension scrapes and syncs'); + const { ext } = await seed(); + + console.log('[build] production bundle against the emulators'); + build(); + const server = await preview(); + const consoleErrors = []; + let browser; + try { + browser = await chromium.launch({ executablePath: process.env.QA_CHROMIUM || undefined }); + const { ctx, page } = await newPage(browser, consoleErrors); + + console.log('[board] all sources, latest'); + await signIn(page, EMAIL, PASSWORD); + await page.waitForSelector('.board-row', { timeout: 40000 }); + let sub = await waitSubtitle(page, /of 1,050 products loaded/); + check('header gives the true total', sub.includes(`200 of ${fmt(UNIQUE)} products loaded`), sub); + await checkColumns(page, 'all sources', { deltas: 0 }); + check('no password card for an email account', (await page.locator('.extpw').count()) === 0); + await page.screenshot({ path: resolve(SHOTS, 'board.png') }); + + await page.click('.board-more button'); + sub = await waitSubtitle(page, /400 of 1,050/); + check('Load more pages on with a cursor', sub.includes(`400 of ${fmt(UNIQUE)}`), sub); + + const lookFor = asinOf(3); // storefront only, not on the loaded pages + await page.fill('.board-search', lookFor); + await page.waitForSelector(`.board-row:has-text("${lookFor}")`, { timeout: 15000 }); + check('an exact ASIN is found past the loaded pages', (await page.locator('.board-row').count()) === 1); + await page.fill('.board-search', ''); + + console.log('[export] whole scope'); + const [download] = await Promise.all([ + page.waitForEvent('download', { timeout: 60000 }), + page.click('button:has-text("Export XLSX")'), + ]); + const file = resolve(SHOTS, download.suggestedFilename()); + await download.saveAs(file); + const sheet = XLSX.read(readFileSync(file)).Sheets['ProScan Leads']; + const table = XLSX.utils.sheet_to_json(sheet, { defval: null }); + check('export writes every product, not the loaded pages', table.length === UNIQUE, `${table.length} rows`); + const filled = (key) => share(table, (r) => r[key] !== null && r[key] !== ''); + check('export: names', filled('Product Name') === 1); + check('export: prices', filled('Current Price') >= 0.85, `${Math.round(filled('Current Price') * 100)}%`); + check('export: ranks', filled('Rank') === 1); + check('export: source ids', filled('Source IDs') === 1); + check('export: last scanned', filled('Last Scanned') === 1); + + console.log('[board] one source: deltas against its previous run'); + await page.selectOption('.board-select', 's_A3K9XELT4QZ6M2'); + sub = await waitSubtitle(page, /deltas against the run of/); + check('storefront scope counts all its products', sub.includes(`200 of ${fmt(STORE_N)} products loaded`), sub); + await checkColumns(page, 'storefront', { deltas: 0.5 }); + await page.screenshot({ path: resolve(SHOTS, 'board-storefront.png') }); + + console.log('[movers] run against run'); + await page.click('.board-seg__btn:has-text("Movers")'); + sub = await waitSubtitle(page, /movers · run of/); + check('storefront movers are counted run against run', sub.includes(`${fmt(expectedMovers)} movers`), `${sub} (want ${expectedMovers})`); + const movers = await checkColumns(page, 'movers', { deltas: 1 }); + const firstDelta = movers.rows[0]?.[header(movers.headers, 'Δ Price')] ?? ''; + check('the biggest drop is on top', firstDelta.startsWith('▼'), firstDelta); + await page.screenshot({ path: resolve(SHOTS, 'movers.png') }); + + await page.selectOption('.board-select', ''); + sub = await waitSubtitle(page, /across 2 sources/); + check('movers across all sources', sub.includes(`${fmt(expectedMovers)} movers across 2 sources`), sub); + + console.log('[drawer]'); + await page.goto(`${BASE}?asin=${asinOf(9)}`); + await page.waitForSelector('.drawer', { timeout: 20000 }); + await page.waitForSelector('.recharts-surface', { timeout: 20000 }); + check('drawer draws the price history', (await page.locator('.drawer [role="alert"]').count()) === 0); + await page.keyboard.press('Escape'); + + console.log('[runs] and [watchlist]'); + await page.click('.nav-link:has-text("Runs")'); + await page.waitForSelector('.run-card:not(.run-card--skeleton)', { timeout: 20000 }); + sub = await waitSubtitle(page, /3 runs/); + check('three run cards', (await page.locator('.run-card:not(.run-card--skeleton)').count()) === 3, sub); + await page.click('.nav-link:has-text("Watchlist")'); + await page.waitForSelector('.wl-row', { timeout: 20000 }); + check('two sources on the watchlist', (await page.locator('.wl-row').count()) === 2); + await ctx.close(); + + console.log('[F-50] a Google account sets a password for the extension'); + const googleErrors = []; + await googlePassword(browser, googleErrors, ext); + // The emulator refuses to link the account's own email with a 400 and + // the dashboard falls back to updatePassword; nothing else may log. + consoleErrors.push(...googleErrors.filter((t) => !/status of 400/.test(t))); + } finally { + await browser?.close().catch(() => {}); + kill(server); + } + + const noise = consoleErrors.filter((t) => !/favicon/i.test(t)); + check('no console errors', noise.length === 0, noise.slice(0, 3).join(' | ')); +} + +async function googlePassword(browser, consoleErrors, ext) { + const email = `google-${Date.now()}@proscan.test`; + const { ctx, page } = await newPage(browser, consoleErrors); + // A Google sign-in, made in Node against the Auth emulator and handed to + // the page as its saved session. The emulator's popup relays through + // Google's gapi script from apis.google.com, which the test must not need. + const idp = firebase('e2e-google-idp'); + const token = JSON.stringify({ sub: `g${Date.now()}`, email, email_verified: true, name: 'Google Tester' }); + const google = await signInWithCredential(idp.auth, GoogleAuthProvider.credential(token)); + check('the account signs in with Google only', google.user.providerData.map((p) => p.providerId).join() === 'google.com'); + const saved = { key: 'firebase:authUser:demo-key:[DEFAULT]', value: google.user.toJSON() }; + await terminate(idp.db); + await deleteApp(idp.app); + + await page.goto(BASE); + await page.waitForSelector('.auth__cta', { timeout: 30000 }); + await page.evaluate( + ({ key, value }) => + new Promise((done, fail) => { + const req = indexedDB.open('firebaseLocalStorageDb'); + req.onerror = () => fail(req.error); + req.onsuccess = () => { + const tx = req.result.transaction('firebaseLocalStorage', 'readwrite'); + tx.objectStore('firebaseLocalStorage').put({ fbase_key: key, value }); + tx.oncomplete = () => done(); + tx.onerror = () => fail(tx.error); + }; + }), + saved, + ); + await page.reload(); + const offered = await page.waitForSelector('.extpw', { timeout: 30000 }).then(() => true, () => false); + if (!offered) { + await page.screenshot({ path: resolve(SHOTS, 'google-signin-failed.png') }); + const text = (await page.locator('body').innerText()).slice(0, 300).replace(/\s+/g, ' '); + check('a Google account is offered a password for the extension', false, text); + await ctx.close(); + return; + } + check('a Google account is offered a password for the extension', (await page.locator('#extpw-title').count()) === 1); + await page.screenshot({ path: resolve(SHOTS, 'extension-password.png') }); + + await page.click('.extpw button:has-text("Set a password")'); + await page.fill('#extpw-new', 'short'); + await page.fill('#extpw-confirm', 'short'); + await page.click('.extpw button:has-text("Save password")'); + check('a short password is refused', (await page.locator('.extpw__error').innerText()).includes('8 characters')); + const password = 'google-then-pass-1'; + await page.fill('#extpw-new', password); + await page.fill('#extpw-confirm', password); + await page.click('.extpw button:has-text("Save password")'); + const linked = await page.waitForSelector('.extpw--done', { timeout: 20000 }).then(() => true, () => false); + check('the password is linked', linked, linked ? '' : await page.locator('.extpw').innerText().catch(() => 'no card')); + if (!linked) { + await page.screenshot({ path: resolve(SHOTS, 'extension-password-failed.png') }); + await ctx.close(); + return; + } + + // The extension signs in with email and password: it now can, as the same account. + const fb = firebase('e2e-google'); + const cred = await signInWithEmailAndPassword(fb.auth, email, password); + const googleUid = await page.evaluate(() => document.querySelector('.sidebar__email')?.textContent ?? ''); + check('the extension can sign in with it', !!cred.user.uid && googleUid.includes(email), googleUid); + const install = installExtension(ext, { uid: cred.user.uid, db: fb.db }); + await install.scrape(KEYWORD, Date.parse('2026-09-12T10:00:00Z'), cards(5, { offset: 5000 }), 48); + await install.flush(); + await terminate(fb.db); + await deleteApp(fb.app); + + await page.reload(); + await page.waitForSelector('.board-row', { timeout: 30000 }); + const sub = await waitSubtitle(page, /products/); + check("and its scans reach the Google account's dashboard", sub.startsWith('5 products'), sub); + await ctx.close(); +} + +const timer = setTimeout(() => { + console.error('[e2e] over the time limit'); + process.exit(1); +}, DEADLINE - Date.now()); + +try { + await main(); +} catch (e) { + console.error('[fatal]', e); + failed++; +} finally { + clearTimeout(timer); + if (known) console.log(`\n${known} known failure(s), see above`); + console.log(failed === 0 ? '\nRESULT: PASS' : `\nRESULT: FAIL (${failed})`); + process.exit(failed === 0 ? 0 : 1); +} diff --git a/scripts/lib/docs.mjs b/scripts/lib/docs.mjs new file mode 100644 index 0000000..3f69de3 --- /dev/null +++ b/scripts/lib/docs.mjs @@ -0,0 +1,25 @@ +// Schema-valid documents for rules tests that need a product in place +// before they probe something else. Checked with the vendored schema, so a +// schema change breaks these loudly instead of quietly. + +import { Timestamp } from 'firebase/firestore'; +import { SV, MK, assertValid, dayKeyOf, runIdOf } from '../../packages/schema/index.js'; + +/** A product as sync writes it after one run of `sourceId`. */ +export function productDoc(asin, { cents = 1999, atMs = Date.parse('2026-09-01T12:00:00Z'), sourceId = 'k_qa' } = {}) { + const runId = runIdOf(sourceId, atMs); + const doc = { + sv: SV, + asin, + mk: MK, + name: `QA product ${asin}`, + url: `https://www.amazon.com/dp/${asin}`, + latest: { p: cents, r: 4.5, v: 120, at: Timestamp.fromMillis(atMs), runId, dayKey: dayKeyOf(atMs) }, + prev: null, + delta: null, + sourceIds: [sourceId], + firstSeenAt: Timestamp.fromMillis(atMs), + firstRunId: runId, + }; + return assertValid('product', doc); +} diff --git a/scripts/lib/extension-sync.mjs b/scripts/lib/extension-sync.mjs new file mode 100644 index 0000000..97bf538 --- /dev/null +++ b/scripts/lib/extension-sync.mjs @@ -0,0 +1,160 @@ +// Drives the extension's real run engine and sync module from its checkout: +// scraped pages go into the engine (IndexedDB via fake-indexeddb), and +// sync.js drains the outbox into Firestore under this repo's rules. Used by +// the sync rules test and the e2e seed, so neither writes hand-made data. +// +// sync.js takes its Firestore functions as a dependency, so it is handed +// this repo's firebase copy and writes through the caller's `db`. + +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + doc, + getDoc, + writeBatch, + Timestamp, + arrayUnion, + deleteField, + FieldPath, +} from 'firebase/firestore'; + +const FIRESTORE = { doc, getDoc, writeBatch, Timestamp, arrayUnion, deleteField, FieldPath }; +const quiet = { log() {}, warn() {}, error() {} }; + +/** Loads the engine, db, sync and schema modules from an extension checkout. */ +export async function loadExtension(extDir) { + const require = createRequire(resolve(extDir, 'package.json')); + await import(pathToFileURL(require.resolve('fake-indexeddb/auto')).href); + const DB = require(resolve(extDir, 'scripts/background/db.js')); + const { createEngine } = require(resolve(extDir, 'scripts/background/engine.js')); + const { createSync } = await import(pathToFileURL(resolve(extDir, 'scripts/background/sync.js')).href); + return { DB, createEngine, createSync }; +} + +function memoryArea(init = {}) { + let data = JSON.parse(JSON.stringify(init)); + return { + async get(keys) { + const list = keys == null ? Object.keys(data) : [].concat(keys); + return Object.fromEntries(list.filter((k) => data[k] !== undefined).map((k) => [k, data[k]])); + }, + async set(items) { + Object.assign(data, JSON.parse(JSON.stringify(items))); + }, + async remove(keys) { + [].concat(keys).forEach((k) => delete data[k]); + }, + }; +} + +let installs = 0; + +/** + * One extension install signed in as `uid`, with a tab that answers every + * message. Returns scrape() to run a whole search and flush() to sync it. + */ +export function installExtension(ext, { uid, db, log = quiet }) { + const name = `proscan-web-${process.pid}-${++installs}`; + const clock = { t: 0 }; + let url = ''; + const chrome = { + storage: { + session: memoryArea(), + local: memoryArea({ account: { uid }, settings: { maxPages: 400 } }), + }, + runtime: { lastError: null }, + tabs: { + sendMessage(tabId, message, opts, cb) { + setImmediate(() => cb(message.type === 'PING' ? { ok: true, kind: 'results', url } : { ok: true })); + }, + async update() { + return {}; + }, + async get(id) { + return { id, url }; + }, + }, + }; + const engine = ext.createEngine({ + chrome, + openDb: () => ext.DB.open({ name }), + now: () => clock.t, + random: () => 0, + setTimer: () => 0, + clearTimer: () => {}, + flags: { CLOUD_SYNC: true }, + log, + }); + const sender = { tab: { id: 1 } }; + + async function reportPage(runId, pageNo, cards, { last, total }) { + if (pageNo > 1) { + clock.t += 5000; + await engine.tick(); + } + const at = new Date(clock.t).toISOString(); + const result = { + kind: last ? 'last' : 'results', + // Rank within the page; the engine makes it run-wide. + products: cards.map((p, j) => ({ + ...p, + placements: p.placements ?? [{ position: j + 1, sponsored: false, rank: j + 1 }], + scrapedAt: at, + })), + placements: cards.length, + nextHref: last ? null : `${url}&page=${pageNo + 1}`, + total, + fill: { asin: 1, title: 1, price: 1 }, + }; + const pageUrl = pageNo === 1 ? url : `${url}&page=${pageNo}`; + const resp = await engine.pageResult({ runId, page: pageNo, url: pageUrl, result }, sender); + if (!resp.ok) throw new Error(`page ${pageNo} refused: ${JSON.stringify(resp)}`); + } + + /** Scrapes `cards` from the search at `searchUrl`, `perPage` to a page, starting at `startMs`. */ + async function scrape(searchUrl, startMs, cards, perPage = 48) { + clock.t = startMs; + url = searchUrl; + const resp = await engine.start({ tabId: 1 }); + if (!resp.ok) throw new Error(`run refused: ${JSON.stringify(resp)}`); + const pages = Math.ceil(cards.length / perPage); + for (let i = 0; i < pages; i++) { + await reportPage(resp.runId, i + 1, cards.slice(i * perPage, (i + 1) * perPage), { + last: i === pages - 1, + total: cards.length, + }); + } + return resp.runId; + } + + const sync = ext.createSync({ db, openStore: () => engine.db(), fs: FIRESTORE, log }); + return { scrape, flush: () => sync.flush(uid), pending: () => sync.pending(uid) }; +} + +export const asinOf = (i) => `B0${i.toString(36).toUpperCase().padStart(8, '0')}`; + +/** + * Search result cards as the content script reports them. `price(i)` gives + * cents or null for a price that failed to parse. + */ +export function cards(n, { price = (i) => 1000 + i, reviews = (i) => 100 + i, offset = 0 } = {}) { + return Array.from({ length: n }, (_, k) => { + const i = k + offset; + const cents = price(i); + const asin = asinOf(i); + return { + asin, + name: `Test product ${i}`, + price: cents === null ? null : `$${(cents / 100).toFixed(2)}`, + priceCents: cents, + currency: cents === null ? null : 'USD', + rating: 3 + (i % 20) / 10, + reviewCount: reviews(i), + isPrime: i % 2 === 0, + sponsored: false, + url: `https://www.amazon.com/dp/${asin}`, + img: `https://m.media-amazon.com/images/I/test-${i}.jpg`, + }; + }); +} diff --git a/scripts/lib/extension.mjs b/scripts/lib/extension.mjs new file mode 100644 index 0000000..08c368e --- /dev/null +++ b/scripts/lib/extension.mjs @@ -0,0 +1,27 @@ +// Finds the extension checkout next to this repo: PROSCAN_EXT, else +// ../ext or ../AmazonSellerScraper. The schema check, the sync rules test +// and the e2e seed all run the extension's own files from there. + +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; + +export const WEB_ROOT = resolve(import.meta.dirname, '..', '..'); + +export function findExtension(env = process.env) { + const candidates = env.PROSCAN_EXT + ? [resolve(env.PROSCAN_EXT)] + : [resolve(WEB_ROOT, '..', 'ext'), resolve(WEB_ROOT, '..', 'AmazonSellerScraper')]; + return candidates.find((dir) => existsSync(resolve(dir, 'packages', 'schema', 'index.js'))) ?? null; +} + +/** The checkout, or exit: these checks mean nothing without the real files. */ +export function requireExtension(what) { + const dir = findExtension(); + if (dir) return dir; + if (process.env.PROSCAN_ALLOW_NO_EXT === '1') { + console.log(`SKIP ${what}: no extension checkout (PROSCAN_ALLOW_NO_EXT=1)`); + process.exit(0); + } + console.error(`${what} needs the extension checkout. Set PROSCAN_EXT, or PROSCAN_ALLOW_NO_EXT=1 to skip.`); + process.exit(1); +} diff --git a/scripts/qa-adversarial-rules.mjs b/scripts/qa-adversarial-rules.mjs new file mode 100644 index 0000000..1fa4df5 --- /dev/null +++ b/scripts/qa-adversarial-rules.mjs @@ -0,0 +1,152 @@ +// Adversarial rules probes, from the audit's firestore-rules harness: tenant +// isolation, plan and owner forgery, oversized and malformed documents, +// forged timestamps, the append-only trail, owner delete, and the writes +// the dashboard really makes. Two signed-in tenants, web SDK, so rules are +// enforced. Run: npm run test:rules + +import { initializeApp, deleteApp } from 'firebase/app'; +import { getAuth, connectAuthEmulator, createUserWithEmailAndPassword } from 'firebase/auth'; +import { + getFirestore, + connectFirestoreEmulator, + collection, + deleteDoc, + deleteField, + doc, + getDoc, + getDocs, + serverTimestamp, + setDoc, + updateDoc, + Timestamp, +} from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; + +const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST || '127.0.0.1:9099'; +const [FS_HOST, FS_PORT] = (process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:8080').split(':'); + +const apps = []; +function tenant(name) { + const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, name); + apps.push(app); + const auth = getAuth(app); + const db = getFirestore(app); + connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); + connectFirestoreEmulator(db, FS_HOST, Number(FS_PORT)); + return { auth, db }; +} + +let failed = 0; +let count = 0; +async function probe(name, want, fn) { + count++; + let got = 'allow'; + let detail = ''; + try { + await fn(); + } catch (e) { + got = e.code === 'permission-denied' ? 'deny' : 'error'; + detail = ` (${e.code || e.message})`; + } + const ok = got === want; + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}: want ${want}, got ${got}${ok ? '' : detail}`); +} + +const big = 'x'.repeat(900 * 1024); +const future = Timestamp.fromDate(new Date('2099-01-01')); +const past = Timestamp.fromDate(new Date('2001-01-01')); + +try { + const A = tenant('adv-a'); + const B = tenant('adv-b'); + const ua = (await createUserWithEmailAndPassword(A.auth, `adv-a-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const ub = (await createUserWithEmailAndPassword(B.auth, `adv-b-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const a = A.db; + const b = B.db; + + console.log('[bootstrap] the dashboard sign-in writes'); + await probe('users/{uid} profile mirror', 'allow', () => + setDoc(doc(b, 'users', ub), { displayName: 'B', email: 'b@proscan.test', defaultWorkspace: ub, createdAt: serverTimestamp() }, { merge: true }), + ); + await probe('workspace create', 'allow', () => + setDoc(doc(b, 'workspaces', ub), { ownerUid: ub, name: "B's workspace", plan: 'free', settings: { defaultRoiPct: 30 }, createdAt: serverTimestamp() }, { merge: true }), + ); + await setDoc(doc(a, 'workspaces', ua), { ownerUid: ua, plan: 'free', createdAt: serverTimestamp() }); + await setDoc(doc(b, 'workspaces', ub, 'products', 'B0SECRET01'), productDoc('B0SECRET01')); + + console.log('[tenants] isolation'); + await probe('A reads users/B', 'deny', () => getDoc(doc(a, 'users', ub))); + await probe('A reads workspaces/B', 'deny', () => getDoc(doc(a, 'workspaces', ub))); + await probe('A reads a B product', 'deny', () => getDoc(doc(a, 'workspaces', ub, 'products', 'B0SECRET01'))); + await probe('A lists B products', 'deny', () => getDocs(collection(a, 'workspaces', ub, 'products'))); + await probe('A writes B sellers', 'deny', () => setDoc(doc(a, 'workspaces', ub, 'sellers', 'x'), { x: 1 })); + await probe('A squats an unclaimed workspace', 'deny', () => setDoc(doc(a, 'workspaces', 'unclaimedUid123'), { ownerUid: ua, plan: 'free' })); + await probe('A deletes a B product', 'deny', () => deleteDoc(doc(a, 'workspaces', ub, 'products', 'B0SECRET01'))); + await probe('A deletes workspaces/B', 'deny', () => deleteDoc(doc(a, 'workspaces', ub))); + + console.log('[forgery] plan and owner'); + await probe('A sets its plan to pro', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { plan: 'pro' })); + await probe('A changes ownerUid', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { ownerUid: ub })); + await probe('A writes plan and role on users/A', 'deny', () => setDoc(doc(a, 'users', ua), { plan: 'pro', role: 'admin' }, { merge: true })); + await probe('A adds members and seats', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { members: [ub], seats: 999 })); + await probe('A writes a limit counter', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'counters', 'sources'), { count: 0 })); + await probe('A removes its plan field', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { plan: deleteField() })); + await probe('A rewrites workspace createdAt', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { createdAt: past })); + + console.log('[shape] oversized and malformed'); + await probe('900 KB product name', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0BIG00001'), { ...productDoc('B0BIG00001'), name: big })); + await probe('product with junk fields and a bad id', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'NOTANASIN!'), { asin: 'NOTANASIN!', junk: { deep: [1, 2, 3] }, spread: 'lol' })); + await probe('negative price and a bad dayKey', 'deny', () => { + const d = productDoc('B0NEG00001'); + return setDoc(doc(a, 'workspaces', ua, 'products', 'B0NEG00001'), { ...d, latest: { ...d.latest, p: -500, dayKey: 'not-a-date' } }); + }); + await probe('1400-byte product id', 'deny', () => { + const id = 'Z'.repeat(1400); + return setDoc(doc(a, 'workspaces', ua, 'products', id), { asin: id }); + }); + await probe('javascript: product url', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0URL00001'), { ...productDoc('B0URL00001'), url: 'javascript:alert(1)' })); + await probe('history d as a 900 KB string', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOEXIST1', 'history', 'daily'), { sv: 1, asin: 'B0NOEXIST1', d: big })); + await probe('event with a 900 KB payload', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOEXIST1', 'events', 'e1'), { type: 'x', blob: big, at: serverTimestamp() })); + await probe('900 KB sellers doc', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sellers', 'S1'), { blob: big })); + await probe('arbitrary alert webhook', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'alerts', 'a1'), { webhook: 'http://evil' })); + await probe('run with forged counters', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'runs', 'r1'), { status: 'complete', dayKey: 'x', sourceId: 'y', counters: { uniqueAsins: -99999, placements: 'many' } })); + await probe('source cadenceDays 0', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_s1'), { type: 'keyword', cadenceDays: 0 })); + await probe('page chunk expiring in 9999', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'runs', 'k_x_1', 'pages', 'p0001'), { sv: 1, runId: 'k_x_1', page: 1, scrapedAt: Timestamp.now(), expireAt: Timestamp.fromDate(new Date('9999-01-01')), count: 0, placements: 0, kind: 'results', items: {} })); + + console.log('[time] forged timestamps'); + await probe('latest.at in 2099', 'deny', () => { + const d = productDoc('B0FUTURE01'); + return setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), { ...d, latest: { ...d.latest, at: future } }); + }); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), productDoc('B0FUTURE01')); + await probe('event backdated to 2001', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e2'), { type: 'stageChange', from: 'new', to: 'purchased', at: past })); + await probe('lead.stageChangedAt forged', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), { lead: { stage: 'approved', stageChangedAt: future } }, { merge: true })); + + console.log('[events] append-only, owner may delete'); + await probe('event stamped by the server', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'), { type: 'stageChange', from: 'new', to: 'reviewing', at: serverTimestamp() })); + await probe('event overwrite', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'), { type: 'rewritten', at: serverTimestamp() })); + await probe('owner deletes an event', 'allow', () => deleteDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'))); + await probe('owner deletes a product', 'allow', () => deleteDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'))); + + console.log('[dashboard] real write shapes'); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), productDoc('B0NOLEAD01')); + await probe('notes on a product with no stage yet', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { notes: 'call supplier' } }, { merge: true })); + await probe('stage move then notes', 'allow', async () => { + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { stage: 'reviewing', stageChangedAt: serverTimestamp() } }, { merge: true }); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { notes: 'second look' } }, { merge: true }); + }); + await probe('stage move on an ASIN with no product', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0GHOST001'), { lead: { stage: 'reviewing', stageChangedAt: serverTimestamp() } }, { merge: true })); + await setDoc(doc(a, 'workspaces', ua, 'sources', 'k_known'), { sv: 1, sourceId: 'k_known', type: 'keyword', sellerId: null, keyword: 'known', url: null, lastRunId: 'k_known_1', lastScrapedAt: Timestamp.now() }); + await probe('nickname on a scanned source', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_known'), { nickname: 'Known' }, { merge: true })); + await probe('nickname on a source never scanned', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_new'), { nickname: 'x' }, { merge: true })); + await probe('owner deletes its workspace', 'allow', () => deleteDoc(doc(a, 'workspaces', ua))); + await probe('owner deletes its user profile', 'allow', () => deleteDoc(doc(b, 'users', ub))); +} catch (e) { + console.error('[fatal]', e); + failed++; +} finally { + console.log(failed === 0 ? `\n${count} probes. RESULT: PASS` : `\n${count} probes. RESULT: FAIL (${failed})`); + for (const app of apps) await deleteApp(app).catch(() => {}); + process.exit(failed === 0 ? 0 : 1); +} diff --git a/scripts/qa-board-count.mjs b/scripts/qa-board-count.mjs index b412d59..7057d82 100644 --- a/scripts/qa-board-count.mjs +++ b/scripts/qa-board-count.mjs @@ -16,8 +16,8 @@ import { query, where, writeBatch, - Timestamp, } from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'board-count'); const auth = getAuth(app); @@ -42,11 +42,7 @@ try { const batch = writeBatch(db); for (let i = 0; i < N; i++) { const asin = `B0QACNT${String(i).padStart(3, '0')}`; - batch.set(doc(col, asin), { - asin, - sourceIds: [i % 2 ? 's_ODD' : 's_EVEN'], - latest: { p: 1000 + i, dayKey: '2026-09-01', at: Timestamp.fromMillis(1e12 + i) }, - }); + batch.set(doc(col, asin), productDoc(asin, { cents: 1000 + i, atMs: 1e12 + i, sourceId: i % 2 ? 's_ODD' : 's_EVEN' })); } await batch.commit(); diff --git a/scripts/qa-e2e.mjs b/scripts/qa-e2e.mjs index 3c42ff1..2ebff3a 100644 --- a/scripts/qa-e2e.mjs +++ b/scripts/qa-e2e.mjs @@ -1,5 +1,7 @@ // Throwaway integration-QA driver (Playwright, emulator-connected DEV server). // Usage: node scripts/qa-e2e.mjs +// Runs on the `npm run seed:fixture` data, which predates the shared +// schema; the dashboard now hides it. scripts/e2e.mjs replaces this. // - signs in as dev@proscan.test / proscan-dev // - walks every screen, screenshots to .screenshots/qa/ // - functional assertions: drawer URL contract, stage-change persistence, diff --git a/scripts/qa-lead-rules.mjs b/scripts/qa-lead-rules.mjs index 448b752..2d82f16 100644 --- a/scripts/qa-lead-rules.mjs +++ b/scripts/qa-lead-rules.mjs @@ -13,6 +13,7 @@ import { setDoc, serverTimestamp, } from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'lead-rules'); const auth = getAuth(app); @@ -48,13 +49,7 @@ try { const product = (asin) => doc(db, 'workspaces', uid, 'products', asin); // A product as sync.js writes it: no lead map at all. - const seed = (asin) => - setDoc(product(asin), { - asin, - mk: 'US', - name: 'QA lead product', - latest: { p: 1999, dayKey: '2026-09-01' }, - }); + const seed = (asin) => setDoc(product(asin), productDoc(asin)); console.log('[lead] notes on an untriaged product'); await seed('B0QALEAD01'); diff --git a/scripts/qa-sync-rules.mjs b/scripts/qa-sync-rules.mjs index 1eae650..93dcc46 100644 --- a/scripts/qa-sync-rules.mjs +++ b/scripts/qa-sync-rules.mjs @@ -1,33 +1,33 @@ -// Emulator integration QA — proves the ProScan extension's cloud-sync writes -// (scripts/background/sync.js :: syncToCloud) are ACCEPTED by the deployed -// firestore.rules, and that cross-tenant writes are DENIED. +// Sync rules test: the extension's real engine and sync.js (from its +// checkout, see scripts/lib/extension.mjs) write two runs of a storefront +// and a keyword into the emulator, and this repo's firestore.rules judge +// every write. Then every written document is read back and checked +// against the vendored schema, and the rules' create-only and tenant +// checks are probed directly. // -// Uses the firebase WEB SDK (NOT firebase-admin) so security rules are -// ENFORCED on every call — admin bypasses rules and would prove nothing. -// -// Connects ONLY to the local emulator suite. Hosts come from the variables -// `firebase emulators:exec` sets, defaulting to auth 9099 / firestore 8080. -// Run: npm run test:rules +// Web SDK, so rules are enforced. Run: npm run test:rules import { initializeApp, deleteApp } from 'firebase/app'; -import { - getAuth, - connectAuthEmulator, - createUserWithEmailAndPassword, - signInWithEmailAndPassword, -} from 'firebase/auth'; +import { getAuth, connectAuthEmulator, createUserWithEmailAndPassword, signOut } from 'firebase/auth'; import { getFirestore, connectFirestoreEmulator, - writeBatch, + collection, doc, getDoc, - serverTimestamp, + getDocs, + setDoc, + updateDoc, Timestamp, - arrayUnion, } from 'firebase/firestore'; +import * as Schema from '../packages/schema/index.js'; +import { requireExtension } from './lib/extension.mjs'; +import { asinOf, cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const extDir = requireExtension('the sync rules test'); +const ext = await loadExtension(extDir); -const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }); +const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'sync-rules'); const auth = getAuth(app); const db = getFirestore(app); const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST || '127.0.0.1:9099'; @@ -35,291 +35,119 @@ const [FS_HOST, FS_PORT] = (process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:80 connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); connectFirestoreEmulator(db, FS_HOST, Number(FS_PORT)); -const results = []; -const pass = (name, detail = '') => { results.push({ ok: true, name, detail }); console.log(` PASS ${name}${detail ? ' — ' + detail : ''}`); }; -const fail = (name, detail = '') => { results.push({ ok: false, name, detail }); console.log(` FAIL ${name}${detail ? ' — ' + detail : ''}`); }; -const check = (name, cond, detail = '') => (cond ? pass(name, detail) : fail(name, detail)); - -// ── helpers faithful to sync.js shapes ─────────────────────────────────────── -const round1 = (n) => Math.round(n * 10) / 10; -const intOrUndef = (n) => (typeof n === 'number' && Number.isFinite(n) ? Math.round(n) : undefined); -const tsOf = (iso) => Timestamp.fromDate(new Date(iso)); - -function pointFrom(q) { - const pt = { - p: intOrUndef(q.priceCents), - r: typeof q.rating === 'number' && q.rating > 0 ? round1(q.rating) : undefined, - v: typeof q.reviewCount === 'number' && q.reviewCount > 0 ? Math.round(q.reviewCount) : undefined, - pr: q.isPrime ? 1 : 0, - }; - Object.keys(pt).forEach((k) => pt[k] === undefined && delete pt[k]); - return pt; -} -function deltaBlock(q) { - const d = q.delta; - if (!d || d.isNew) return undefined; - const out = {}; - if (typeof d.dPriceCents === 'number') { - out.p = Math.round(d.dPriceCents); - const prev = typeof q.priceCents === 'number' ? q.priceCents - d.dPriceCents : null; - if (prev && prev !== 0) out.pPct = round1((d.dPriceCents / prev) * 100); - } - if (typeof d.dRating === 'number') out.r = round1(d.dRating); - if (typeof d.dReviews === 'number') out.v = Math.round(d.dReviews); - return Object.keys(out).length ? out : undefined; -} - -/** Faithful port of sync.js :: syncToCloud — same four write types/shapes. */ -async function syncToCloud(uid, bundle) { - const queue = (bundle && bundle.syncQueue) || []; - if (!queue.length) return { written: 0, runId: null, products: 0 }; - - const meta = (bundle && bundle.scrapeRunMeta) || {}; - const sourceId = meta.sellerId ? `s_${meta.sellerId}` : meta.keyword - ? `k_${String(meta.keyword).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')}` - : 'k_unknown'; - const startMs = meta.startedAt ? Date.parse(meta.startedAt) : Date.now(); - const runId = `${sourceId}_${startMs}`; - const dayKey = new Date(startMs).toISOString().slice(0, 10); - const mk = 'US'; - - let products = 0; - const batch = writeBatch(db); - for (const q of queue) { - if (!q || !q.asin) continue; - const at = q.scrapedAt ? tsOf(q.scrapedAt) : serverTimestamp(); - const payload = { - asin: q.asin, - mk, - name: q.name, - url: q.url, - latest: { ...pointFrom(q), at, runId, dayKey }, - sourceIds: arrayUnion(sourceId), - }; - const d = deltaBlock(q); - if (d) payload.delta = d; - if (q.delta && q.delta.isNew) { - payload.firstSeenAt = at; - payload.firstRunId = runId; - } - // WRITE 1: products/{asin} - batch.set(doc(db, 'workspaces', uid, 'products', q.asin), payload, { merge: true }); - // WRITE 2: products/{asin}/history/daily - batch.set( - doc(db, 'workspaces', uid, 'products', q.asin, 'history', 'daily'), - { asin: q.asin, d: { [dayKey]: pointFrom(q) } }, - { merge: true }, - ); - products++; - } - await batch.commit(); - - const head = writeBatch(db); - // WRITE 3: runs/{runId} - head.set( - doc(db, 'workspaces', uid, 'runs', runId), - { - runId, - sourceId, - source: { type: meta.type || 'keyword', sellerId: meta.sellerId ?? null, keyword: meta.keyword ?? null, url: meta.url ?? null }, - mk, - dayKey, - startedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - finishedAt: serverTimestamp(), - status: 'complete', - pagesDone: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - pagesPlanned: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - counters: { - placements: queue.length, - uniqueAsins: new Set(queue.map((q) => q.asin)).size, - sponsored: 0, - priceParseFailures: queue.filter((q) => q.priceCents == null).length, - newSeen: queue.filter((q) => q.delta && q.delta.isNew).length, - }, - }, - { merge: true }, - ); - // WRITE 4: sources/{sourceId} - head.set( - doc(db, 'workspaces', uid, 'sources', sourceId), - { - sourceId, - type: meta.type === 'storefront' ? 'storefront' : 'keyword', - sellerId: meta.sellerId ?? null, - keyword: meta.keyword ?? null, - url: meta.url ?? null, - lastRunId: runId, - lastScrapedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - }, - { merge: true }, - ); - await head.commit(); - - return { written: products * 2 + 2, runId, products, sourceId, dayKey }; -} - -// ── sample producer queue: one NEW product + one RETURNING product ─────────── -const bundle = { - scrapeRunMeta: { - type: 'storefront', - sellerId: 'A3K9XELT4QZ6M2', - keyword: null, - url: 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2', - startedAt: '2026-06-21T10:00:00Z', - }, - scrapeRunPages: [{ page: 1 }, { page: 2 }], - syncQueue: [ - { - // RETURNING item — has a delta block (price dropped 200c from 2599 -> 2399) - asin: 'B0QASYNC01', - name: 'QA Returning Tumbler 40oz', - url: 'https://www.amazon.com/dp/B0QASYNC01', - priceCents: 2399, - rating: 4.6, - reviewCount: 1873, - isPrime: true, - scrapedAt: '2026-06-21T10:04:53Z', - delta: { isNew: false, dPriceCents: -200, dRating: 0.1, dReviews: 172 }, - }, - { - // NEW item — first-sight stamps, no delta - asin: 'B0QASYNC02', - name: 'QA New Mixing Bowl Set', - url: 'https://www.amazon.com/dp/B0QASYNC02', - priceCents: 1599, - rating: 4.1, - reviewCount: 233, - isPrime: false, - scrapedAt: '2026-06-21T10:05:10Z', - delta: { isNew: true }, - }, - ], +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` (${detail})`}`); }; - -let exitCode = 0; -try { - console.log('[auth] signing in test user A …'); - const emailA = `qa-sync-A-${Date.now()}@proscan.test`; - let credA; - try { - credA = await createUserWithEmailAndPassword(auth, emailA, 'qa-pass-123'); - } catch (e) { - credA = await signInWithEmailAndPassword(auth, emailA, 'qa-pass-123'); +// A bug that lives in the extension: reported, not fixed here. Strict, so +// the mark has to come off once the extension fixes it. +let known = 0; +const knownFailure = (name, ok, detail = '') => { + if (ok) { + failed++; + console.log(` FAIL ${name} now passes; remove its known-failure mark`); + } else { + known++; + console.log(` KNOWN FAILURE ${name}${detail ? ` (${detail})` : ''}`); } - const uidA = credA.user.uid; - console.log(` uid A = ${uidA}`); - - // ── POSITIVE: sync into the owner's own workspace ────────────────────────── - console.log('\n[positive] syncToCloud(uidA) — faithful sync.js writes'); - let res; +}; +async function denied(name, fn) { + let code = 'allowed'; try { - res = await syncToCloud(uidA, bundle); - pass('all 4 write types committed (rules ACCEPTED)', `products=${res.products}, runId=${res.runId}`); + await fn(); } catch (e) { - fail('writes rejected by rules', `${e.code || ''} ${e.message}`); - throw e; // can't read back if writes failed - } - - // ── READ-BACK each of the 4 write types ──────────────────────────────────── - console.log('\n[read-back] verifying each write type round-trips'); - - // 1 · products/{asin} — returning (B0QASYNC01) with delta - const p1 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC01')); - check('WRITE 1 products/{asin} (returning) exists', p1.exists()); - if (p1.exists()) { - const d = p1.data(); - check(' · asin field == doc id', d.asin === 'B0QASYNC01', `asin=${d.asin}`); - check(' · latest.dayKey is string "2026-06-21"', d.latest?.dayKey === '2026-06-21', `dayKey=${d.latest?.dayKey}`); - check(' · latest.p int == 2399', d.latest?.p === 2399, `p=${d.latest?.p}`); - check(' · latest.v int == 1873', d.latest?.v === 1873, `v=${d.latest?.v}`); - check(' · delta.p == -200, delta.pPct == -7.7', d.delta?.p === -200 && d.delta?.pPct === -7.7, `delta=${JSON.stringify(d.delta)}`); - check(' · sourceIds arrayUnion has s_A3K9XELT4QZ6M2', Array.isArray(d.sourceIds) && d.sourceIds.includes('s_A3K9XELT4QZ6M2'), JSON.stringify(d.sourceIds)); - check(' · returning item has NO firstSeenAt stamp', d.firstSeenAt === undefined); - } - - // 1b · products/{asin} — NEW (B0QASYNC02) with first-sight stamps - const p2 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC02')); - check('WRITE 1 products/{asin} (new) exists', p2.exists()); - if (p2.exists()) { - const d = p2.data(); - check(' · new item stamped firstRunId', d.firstRunId === res.runId, `firstRunId=${d.firstRunId}`); - check(' · new item has firstSeenAt timestamp', d.firstSeenAt instanceof Timestamp); - check(' · new item has NO delta block', d.delta === undefined); - check(' · latest.p int == 1599', d.latest?.p === 1599, `p=${d.latest?.p}`); - } - - // 2 · products/{asin}/history/daily - const h1 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC01', 'history', 'daily')); - check('WRITE 2 products/{asin}/history/daily exists', h1.exists()); - if (h1.exists()) { - const d = h1.data(); - check(' · history.d["2026-06-21"].p == 2399', d.d?.['2026-06-21']?.p === 2399, JSON.stringify(d.d?.['2026-06-21'])); + code = e.code || e.message; } + check(name, code === 'permission-denied', code); +} - // 3 · runs/{runId} - const r1 = await getDoc(doc(db, 'workspaces', uidA, 'runs', res.runId)); - check('WRITE 3 runs/{runId} exists', r1.exists()); - if (r1.exists()) { - const d = r1.data(); - check(' · status enum == "complete"', d.status === 'complete', `status=${d.status}`); - check(' · dayKey is string', typeof d.dayKey === 'string', `dayKey=${d.dayKey}`); - check(' · sourceId is string', typeof d.sourceId === 'string', `sourceId=${d.sourceId}`); - check(' · counters.placements == 2', d.counters?.placements === 2, `placements=${d.counters?.placements}`); - check(' · counters.newSeen == 1', d.counters?.newSeen === 1, `newSeen=${d.counters?.newSeen}`); - } +const DAY = 86400000; +const STORE = 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2&marketplaceID=ATVPDKIKX0DER'; +const KEYWORD = 'https://www.amazon.com/s?k=insulated+tumbler'; - // 4 · sources/{sourceId} - const s1 = await getDoc(doc(db, 'workspaces', uidA, 'sources', res.sourceId)); - check('WRITE 4 sources/{sourceId} exists', s1.exists()); - if (s1.exists()) { - const d = s1.data(); - check(' · type enum == "storefront"', d.type === 'storefront', `type=${d.type}`); - check(' · lastRunId == runId', d.lastRunId === res.runId, `lastRunId=${d.lastRunId}`); - check(' · cadenceDays omitted (rules default 7 is int)', d.cadenceDays === undefined); +try { + console.log(`[sync] extension at ${extDir}`); + const uid = (await createUserWithEmailAndPassword(auth, `qa-sync-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const ws = (...path) => doc(db, 'workspaces', uid, ...path); + const install = installExtension(ext, { uid, db }); + + const day1 = Date.parse('2026-06-02T15:00:00Z'); + const day8 = day1 + 7 * DAY; + const run1 = await install.scrape(STORE, day1, cards(130), 48); + // A week later every 10th price fails to parse and the rest drop by 100 cents. + const run2 = await install.scrape(STORE, day8, cards(130, { price: (i) => (i % 10 === 0 ? null : 900 + i) }), 48); + const run3 = await install.scrape(KEYWORD, day8 + 3600000, cards(20, { offset: 120 }), 48); + + console.log('[sync] flush through the rules'); + const totals = await install.flush(); + check('every outbox entry was written', totals.entries === 3 * 2 + 3 + 1 && (await install.pending()) === 0, JSON.stringify(totals)); + check('three runs, seven pages', totals.runs === 3 && totals.pages === 7, JSON.stringify(totals)); + + console.log('[sync] read back and check against the schema'); + const kinds = []; + for (const runId of [run1, run2, run3]) { + const run = (await getDoc(ws('runs', runId))).data(); + kinds.push(['run', runId, Schema.validateRun(run)]); + const pages = await getDocs(collection(db, 'workspaces', uid, 'runs', runId, 'pages')); + for (const p of pages.docs) kinds.push(['page', p.ref.path, Schema.validatePage(p.data())]); } - - // ── NEGATIVE: cross-tenant write must be DENIED ──────────────────────────── - console.log('\n[negative] cross-tenant write — uidA -> workspaces/{B}/products'); - const uidB = 'someOtherTenantUid_000000000000'; - let denied = false; - let errInfo = ''; - try { - await import('firebase/firestore').then(({ setDoc }) => - setDoc( - doc(db, 'workspaces', uidB, 'products', 'B0QASYNC01'), - { asin: 'B0QASYNC01', mk: 'US', name: 'cross-tenant intrusion', latest: { dayKey: '2026-06-21', p: 1 } }, - { merge: true }, - ), - ); - } catch (e) { - denied = e.code === 'permission-denied' || /permission/i.test(e.message); - errInfo = `${e.code || ''} ${e.message}`; + const products = await getDocs(collection(db, 'workspaces', uid, 'products')); + for (const p of products.docs) kinds.push(['product', p.id, Schema.validateProduct(p.data())]); + for (const id of ['s_A3K9XELT4QZ6M2', 'k_insulated-tumbler']) { + kinds.push(['source', id, Schema.validateSource((await getDoc(ws('sources', id))).data())]); } - check('cross-tenant write DENIED (permission-denied)', denied, denied ? errInfo : 'WRITE WAS ACCEPTED — RULES NOT ENFORCING TENANT ISOLATION'); - - // confirm nothing landed in B's workspace (read it back as A — also denied, - // so a thrown read is itself proof the doc is unreachable / not created) - let crossReadBlockedOrEmpty = false; + kinds.push(['history', asinOf(11), Schema.validateHistory((await getDoc(ws('products', asinOf(11), 'history', 'daily'))).data())]); + const bad = kinds.filter(([, , errs]) => errs.length > 0); + check(`${kinds.length} documents match schema v${Schema.SV}`, bad.length === 0, bad.slice(0, 3).map(([k, id, e]) => `${k} ${id}: ${e[0]}`).join('; ')); + check('140 products, one per ASIN', products.size === 140, `size=${products.size}`); + + const moved = (await getDoc(ws('products', asinOf(11)))).data(); + check('latest is the second run', moved.latest.runId === run2 && moved.latest.p === 911, JSON.stringify(moved.latest)); + check('delta against the first run', moved.delta?.p === -100 && moved.delta?.days === 7, JSON.stringify(moved.delta)); + check('firstSeenAt stays on the first run', moved.firstRunId === run1 && moved.firstSeenAt.toMillis() === day1); + const failedPrice = (await getDoc(ws('products', asinOf(10)))).data(); + check('a price that failed to parse is not kept from last week', !('p' in failedPrice.latest)); + const both = (await getDoc(ws('products', asinOf(125)))).data(); + // sync-plan.js names sourceIds in mergeFields, and a field in the mask is + // replaced, so arrayUnion keeps only the last source. + knownFailure('NEW-SYNC-1 (F-20 area): a product in two sources lists both', both.sourceIds.length === 2, JSON.stringify(both.sourceIds)); + check('rank is written', Number.isInteger(moved.latest.rk), JSON.stringify(moved.latest)); + + const header = (await getDoc(ws('runs', run2))).data(); + check('run header counts the run', header.status === 'complete' && header.counters.uniqueAsins === 130 && header.pagesDone === 3, JSON.stringify(header.counters)); + + console.log('[rules] create-only and forged values'); + await denied('rewrite firstSeenAt', () => updateDoc(ws('products', asinOf(11)), { firstSeenAt: Timestamp.now() })); + await denied('drop firstRunId', () => setDoc(ws('products', asinOf(11)), { ...moved, firstRunId: null })); + await denied('latest.at in 2099', () => updateDoc(ws('products', asinOf(11)), { 'latest.at': Timestamp.fromDate(new Date('2099-01-01')) })); + await denied('price as a string', () => updateDoc(ws('products', asinOf(11)), { 'latest.p': '9.11' })); + await denied('unknown product field', () => updateDoc(ws('products', asinOf(11)), { spread: { sc: 3 } })); + await denied('run id that is not {sourceId}_{startMs}', () => setDoc(ws('runs', 'k_x_1'), { ...header, runId: 'k_x_1' })); + await denied('page chunk under the wrong run', () => setDoc(ws('runs', run1, 'pages', 'p0009'), { sv: 1, runId: run2, page: 9, scrapedAt: Timestamp.now(), expireAt: Timestamp.now(), count: 0, placements: 0, kind: 'results', items: {} })); + + console.log('[rules] another account'); + await signOut(auth); + await createUserWithEmailAndPassword(auth, `qa-sync-b-${Date.now()}@proscan.test`, 'qa-pass-123'); + const intruder = installExtension(ext, { uid, db }); + await intruder.scrape(KEYWORD, day8 + 7200000, cards(3), 48); + let code = 'allowed'; try { - const bdoc = await getDoc(doc(db, 'workspaces', uidB, 'products', 'B0QASYNC01')); - crossReadBlockedOrEmpty = !bdoc.exists(); + await intruder.flush(); } catch (e) { - crossReadBlockedOrEmpty = true; // read denied too — also fine + code = e.code; } - check('cross-tenant doc absent/unreadable for uidA', crossReadBlockedOrEmpty); + check('sync into another workspace is refused', code === 'permission-denied', code); + check('and its entries stay queued', (await intruder.pending()) === 2); + await denied('reading another workspace', () => getDoc(ws('products', asinOf(11)))); } catch (e) { - console.error('\n[fatal]', e); - exitCode = 1; + console.error('[fatal]', e); + failed++; } finally { - // ── summary ──────────────────────────────────────────────────────────────── - const failed = results.filter((r) => !r.ok); - console.log('\n──────── SUMMARY ────────'); - console.log(`assertions: ${results.length} passed: ${results.length - failed.length} failed: ${failed.length}`); - if (failed.length) { - for (const f of failed) console.log(` FAILED: ${f.name} ${f.detail}`); - exitCode = 1; - } - console.log(exitCode === 0 ? '\nRESULT: PASS' : '\nRESULT: FAIL'); - try { await deleteApp(app); } catch {} - process.exit(exitCode); + if (known) console.log(`\n${known} known failure(s), see above`); + console.log(failed === 0 ? '\nRESULT: PASS' : `\nRESULT: FAIL (${failed})`); + try { + await deleteApp(app); + } catch {} + process.exit(failed === 0 ? 0 : 1); } diff --git a/scripts/seed-emulators.mjs b/scripts/seed-emulators.mjs index 9020430..db5da5e 100644 --- a/scripts/seed-emulators.mjs +++ b/scripts/seed-emulators.mjs @@ -1,5 +1,10 @@ // ProScan emulator seed fixtures — see docs/architecture/data-model.md §4. // +// Predates the shared schema (packages/schema): these documents carry no +// `sv` and include Phase 5 fields, so the dashboard now leaves them out +// with a schema notice. `npm run seed` writes real extension output. +// Run this one with `npm run seed:fixture`. +// // SAFETY: this script may only ever talk to the local Firebase Emulator Suite. // It defaults the emulator host env vars and hard-fails if either is empty, so // writes against production Firestore/Auth are impossible by construction. diff --git a/scripts/seed-extension.mjs b/scripts/seed-extension.mjs new file mode 100644 index 0000000..ad4e27a --- /dev/null +++ b/scripts/seed-extension.mjs @@ -0,0 +1,48 @@ +// Dev seed through the extension: its real engine and sync.js scan a +// storefront twice a week apart and a keyword once, into the running +// emulators, as dev@proscan.test / proscan-dev. Unlike seed-emulators.mjs +// (firebase-admin fixtures with Phase 5 fields), this is exactly what the +// extension writes. +// +// npm run emulators (one terminal) +// npm run seed (another) + +process.env.FIRESTORE_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_FIRESTORE_PORT || 8080}`; +process.env.FIREBASE_AUTH_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_AUTH_PORT || 9099}`; + +import { initializeApp, deleteApp } from 'firebase/app'; +import { + getAuth, + connectAuthEmulator, + createUserWithEmailAndPassword, + signInWithEmailAndPassword, +} from 'firebase/auth'; +import { getFirestore, connectFirestoreEmulator, terminate } from 'firebase/firestore'; +import { requireExtension } from './lib/extension.mjs'; +import { cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const EMAIL = 'dev@proscan.test'; +const PASSWORD = 'proscan-dev'; +const DAY = 86400000; + +const ext = await loadExtension(requireExtension('the extension seed')); +const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }); +const auth = getAuth(app); +const db = getFirestore(app); +connectAuthEmulator(auth, `http://${process.env.FIREBASE_AUTH_EMULATOR_HOST}`, { disableWarnings: true }); +const [host, port] = process.env.FIRESTORE_EMULATOR_HOST.split(':'); +connectFirestoreEmulator(db, host, Number(port)); + +const cred = await signInWithEmailAndPassword(auth, EMAIL, PASSWORD).catch(() => + createUserWithEmailAndPassword(auth, EMAIL, PASSWORD), +); +const install = installExtension(ext, { uid: cred.user.uid, db }); +const start = Date.now() - 8 * DAY; +const moved = (i) => (i % 10 === 0 ? null : i % 3 === 0 ? 1000 + i - 50 * ((i % 7) + 1) : 1000 + i); +await install.scrape('https://www.amazon.com/s?me=A3K9XELT4QZ6M2', start, cards(300), 48); +await install.scrape('https://www.amazon.com/s?me=A3K9XELT4QZ6M2', start + 7 * DAY, cards(300, { price: moved }), 48); +await install.scrape('https://www.amazon.com/s?k=stainless+tumbler', start + 7 * DAY + 3600000, cards(120, { offset: 280 }), 48); +console.log('synced', await install.flush()); +await terminate(db); +await deleteApp(app); +process.exit(0);