From 749123792dfd3d55cb1496cda44d22ec2d0ebacb Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:02:40 -0400 Subject: [PATCH 01/28] Vendor the extension's cloud schema and check it stays identical --- package.json | 3 +- packages/schema/index.d.ts | 150 ++++++++++++ packages/schema/index.js | 480 +++++++++++++++++++++++++++++++++++++ scripts/check-schema.mjs | 28 +++ scripts/lib/extension.mjs | 27 +++ 5 files changed, 687 insertions(+), 1 deletion(-) create mode 100644 packages/schema/index.d.ts create mode 100644 packages/schema/index.js create mode 100644 scripts/check-schema.mjs create mode 100644 scripts/lib/extension.mjs diff --git a/package.json b/package.json index 1a3e3a7..7020719 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", - "test": "npm run typecheck && npm run build && node scripts/check-bundle.mjs && npm run test:rules" + "check:schema": "node scripts/check-schema.mjs", + "test": "npm run typecheck && npm run check:schema && npm run build && node scripts/check-bundle.mjs && npm run test:rules" }, "dependencies": { "@fontsource-variable/archivo": "^5.2.8", diff --git a/packages/schema/index.d.ts b/packages/schema/index.d.ts new file mode 100644 index 0000000..7b34172 --- /dev/null +++ b/packages/schema/index.d.ts @@ -0,0 +1,150 @@ +// Types for index.js, the cloud schema vendored from the extension +// (packages/schema/index.js there). index.js is copied byte for byte; +// this file is ours. scripts/check-schema.mjs keeps the two in step. + +export declare const SV: 1; +export declare const MK: 'US'; +export declare const ASIN_RE: RegExp; +export declare const DAY_KEY_RE: RegExp; +export declare const PAGE_TTL_DAYS: number; +export declare const MAX_PAGE_ITEMS: number; +export declare const RUN_STATUS: readonly ['active', 'complete', 'stopped', 'dead']; +export declare const SOURCE_TYPES: readonly ['storefront', 'keyword']; + +export type RunStatus = (typeof RUN_STATUS)[number]; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** Anything timeMs understands: ms, a Date, or a Timestamp. */ +export type TimeLike = number | Date | { toMillis(): number }; + +/** One observation. Unknown values are left out, never null or 0. */ +export interface Point { + /** price, cents */ + p?: number; + /** rating, 0 to 5 */ + r?: number; + /** review count */ + v?: number; + pr?: 0 | 1; + /** organic rank within the run, from 1 */ + rk?: number; + /** sponsored card, page items only */ + sp?: 0 | 1; +} + +export interface Delta { + p?: number; + pPct?: number; + r?: number; + v?: number; + days?: number; +} + +export interface Source { + type: SourceType; + sellerId: string | null; + keyword: string | null; + url: string | null; +} + +export interface SourceDoc { + sv: number; + sourceId: string; + type: SourceType; + sellerId: string | null; + keyword: string | null; + url: string | null; + lastRunId: string; + lastScrapedAt: T; + catalogSize?: number; +} + +export interface RunCounters { + placements: number; + uniqueAsins: number; + sponsored: number; + priceParseFailures: number; + newSeen: number; +} + +export interface RunDoc { + sv: number; + runId: string; + sourceId: string; + source: Source; + mk: string; + dayKey: string; + startedAt: T; + finishedAt: T | null; + status: RunStatus; + reason: string | null; + pagesDone: number; + maxPages: number; + pagesPlanned: number | null; + totalResultsOnSerp: number | null; + counters: RunCounters; +} + +export interface PageDoc { + sv: number; + runId: string; + page: number; + scrapedAt: T; + expireAt: T; + count: number; + placements: number; + kind: string; + items: Record; + /** result count Amazon showed; sync-plan.js writes it, the typedef there omits it */ + total?: number; + truncated?: boolean; +} + +export interface ProductDoc { + sv: number; + asin: string; + mk: string; + name?: string | null; + url: string; + img?: string | null; + latest: Point & { at: T; runId: string; dayKey: string }; + prev: (Point & { at?: T }) | null; + delta: Delta | null; + sourceIds: string[]; + firstSeenAt?: T; + firstRunId?: string; +} + +export interface HistoryDoc { + sv: number; + asin: string; + d: Record; +} + +export declare function hash(s: string): string; +export declare function slugify(s: string): string; +export declare function sourceOf(url: string): Source; +export declare function sourceIdOf(source: Source): string; +export declare function runIdOf(sourceId: string, startMs: number): string; +export declare function pageIdOf(page: number): string; +export declare function dayKeyOf(ms: number, tzOffsetMin?: number): string; +export declare function expireAtMs(startMs: number): number; +export declare function runStatusOf(state: string): RunStatus; +export declare function timeMs(v: unknown): number | null; +export declare function pointOf(rec: { + priceCents?: number | null; + rating?: number | null; + reviewCount?: number | null; + isPrime?: boolean; + organicRank?: number | null; +}): Point; +export declare function deltaOf(now: Point, prev: Point | null, days?: number | null): Delta | null; + +export declare function validateSource(doc: unknown): string[]; +export declare function validateRun(doc: unknown): string[]; +export declare function validatePage(doc: unknown): string[]; +export declare function validateProduct(doc: unknown): string[]; +export declare function validateHistory(doc: unknown): string[]; + +export type DocKind = 'source' | 'run' | 'page' | 'product' | 'history'; +export declare function assertValid(kind: DocKind, doc: D): D; diff --git a/packages/schema/index.js b/packages/schema/index.js new file mode 100644 index 0000000..edb605e --- /dev/null +++ b/packages/schema/index.js @@ -0,0 +1,480 @@ +/** + * @fileoverview The ProScan cloud schema: types, validators and id builders. + * + * One file shared by the extension (which writes) and the dashboard (which + * reads). It has no imports and no Firebase code, so the dashboard can copy + * it as is; keep the copies byte-identical and bump SV on any change to a + * document's shape. + * + * Conventions: + * - Money is integer cents. + * - Unknown is null, never 0. In compact points (latest, prev, history + * days, page items) an unknown value is left out, so read it as + * `point.p ?? null`. The Firestore rules check `p is int` when present. + * - Every document carries `sv`, the schema version it was written with. + * - Times are Firestore Timestamps in the cloud. Builders and validators + * here take anything `timeMs` understands: a number of ms, a Date, or an + * object with toMillis(). + * + * Paths, all under workspaces/{uid}: + * sources/{sourceId} SourceDoc + * runs/{runId} RunDoc + * runs/{runId}/pages/{pageId} PageDoc + * products/{asin} ProductDoc (extension fields; lead, + * tags and verdict are the dashboard's) + * products/{asin}/history/daily HistoryDoc + * + * @module Schema + */ + +/** Schema version written into every document. */ +export const SV = 1; + +/** The one marketplace so far. */ +export const MK = 'US'; + +export const ASIN_RE = /^[A-Z0-9]{10}$/; +export const DAY_KEY_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Page chunks expire this long after the run started (TTL policy on expireAt). */ +export const PAGE_TTL_DAYS = 400; + +/** The rules cap a page chunk's items map at this size. */ +export const MAX_PAGE_ITEMS = 120; + +export const RUN_STATUS = ['active', 'complete', 'stopped', 'dead']; +export const SOURCE_TYPES = ['storefront', 'keyword']; + +/** + * @typedef {Object} Point one observation of a product + * @property {number} [p] price, cents + * @property {number} [r] rating, 0 to 5, one decimal + * @property {number} [v] review count + * @property {0|1} [pr] Prime badge + * @property {number} [rk] organic rank within the run, from 1 + * @property {0|1} [sp] sponsored card (page items only) + */ + +/** + * @typedef {Object} Delta change against the previous observation + * @property {number} [p] price change, cents + * @property {number} [pPct] price change, percent of the previous price + * @property {number} [r] rating change + * @property {number} [v] review count change + * @property {number} [days] days between the two observations + */ + +/** + * @typedef {Object} Source + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + */ + +/** + * @typedef {Object} SourceDoc + * @property {number} sv + * @property {string} sourceId + * @property {'storefront'|'keyword'} type + * @property {?string} sellerId + * @property {?string} keyword + * @property {?string} url + * @property {string} lastRunId + * @property {*} lastScrapedAt + * @property {number} [catalogSize] result count Amazon showed, when known + */ + +/** + * @typedef {Object} RunDoc + * @property {number} sv + * @property {string} runId + * @property {string} sourceId + * @property {Source} source + * @property {string} mk + * @property {string} dayKey local date the run started, YYYY-MM-DD + * @property {*} startedAt + * @property {*} finishedAt null while active + * @property {'active'|'complete'|'stopped'|'dead'} status + * @property {?string} reason how it ended, in the extension's words + * @property {number} pagesDone + * @property {number} maxPages + * @property {?number} pagesPlanned known only once a run completes + * @property {?number} totalResultsOnSerp + * @property {{placements:number, uniqueAsins:number, sponsored:number, priceParseFailures:number, newSeen:number}} counters + */ + +/** + * @typedef {Object} PageDoc + * @property {number} sv + * @property {string} runId + * @property {number} page + * @property {*} scrapedAt + * @property {*} expireAt + * @property {number} count products first seen on this page + * @property {number} placements cards on this page + * @property {string} kind results or last + * @property {Object} items every ASIN on the page + * @property {boolean} [truncated] more than MAX_PAGE_ITEMS ASINs + */ + +/** + * @typedef {Object} ProductDoc + * @property {number} sv + * @property {string} asin + * @property {string} mk + * @property {?string} name + * @property {string} url + * @property {?string} img + * @property {Point & {at:*, runId:string, dayKey:string}} latest + * @property {?(Point & {at:*})} prev + * @property {?Delta} delta + * @property {string[]} sourceIds + * @property {*} [firstSeenAt] set once, when the document is created + * @property {string} [firstRunId] + */ + +/** + * @typedef {Object} HistoryDoc + * @property {number} sv + * @property {string} asin + * @property {Object} d dayKey to point + */ + +// ── Ids ───────────────────────────────────────────────────────── + +/** 32-bit FNV-1a of a string, base 36. */ +export function hash(s) { + let h = 0x811c9dc5; + for (const ch of String(s)) { + const c = ch.codePointAt(0); + h ^= c & 0xff; + h = Math.imul(h, 0x01000193); + if (c > 0xff) { + h ^= c >>> 8; + h = Math.imul(h, 0x01000193); + } + } + return (h >>> 0).toString(36); +} + +export function slugify(s) { + return String(s).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); +} + +const SELLER_RE = /^[A-Z0-9]{6,20}$/; +// Query params that change per visit, not per search. +const NOISE = new Set(['page', 'ref', 'qid', 'xpid', 'crid', 'sprefix', 'dib', 'dib_tag', 'sr', 'ds', 'pd_rd_r', 'pf_rd_r']); + +/** The seller in a search URL: me=, seller=, or p_6:/me: inside rh=. */ +function sellerIn(params) { + const direct = params.get('me') || params.get('seller'); + if (direct) return direct.toUpperCase(); + const m = (params.get('rh') || '').match(/(?:^|,)(?:p_6|me):([A-Za-z0-9]+)/); + return m ? m[1].toUpperCase() : null; +} + +/** + * What an Amazon search URL scans. + * @param {string} url + * @returns {Source} + */ +export function sourceOf(url) { + let params = null; + try { + params = new URL(url).searchParams; + } catch (e) { + return { type: 'keyword', sellerId: null, keyword: null, url: url || null }; + } + const seller = sellerIn(params); + if (seller && SELLER_RE.test(seller)) { + return { type: 'storefront', sellerId: seller, keyword: params.get('k') || null, url }; + } + const keyword = params.get('k') || params.get('field-keywords') || null; + return { type: 'keyword', sellerId: null, keyword: keyword ? keyword.trim() : null, url }; +} + +/** + * The source id: s_{sellerId} for a storefront, k_{slug} for a keyword. + * A keyword with non-ASCII letters, or one too long, keeps what it can of + * the slug and adds a hash, so two keywords never share an id. A search + * with neither gets k_x{hash} of its query. + * @param {Source} source + * @returns {string} + */ +export function sourceIdOf(source) { + if (source && source.type === 'storefront' && source.sellerId) return `s_${source.sellerId}`; + const kw = source && source.keyword ? String(source.keyword).normalize('NFC').trim().toLowerCase() : ''; + if (kw) { + const slug = slugify(kw); + // eslint-disable-next-line no-control-regex + const plain = !/[^\x00-\x7f]/.test(kw) && slug.length > 0 && slug.length <= 80; + return plain ? `k_${slug}` : `k_${slug ? slug.slice(0, 60) + '-' : ''}h${hash(kw)}`; + } + let query = ''; + try { + const u = new URL(source.url); + query = [...u.searchParams.entries()] + .filter(([k]) => !NOISE.has(k)) + .map(([k, v]) => `${k}=${v}`) + .sort() + .join('&'); + query = u.pathname + '?' + query; + } catch (e) { /* no URL */ } + return query ? `k_x${hash(query)}` : 'k_unknown'; +} + +/** Run id: {sourceId}_{startMs}, minted once when the run starts. */ +export function runIdOf(sourceId, startMs) { + return `${sourceId}_${startMs}`; +} + +/** Page chunk id: p0001 for page 1. */ +export function pageIdOf(page) { + return 'p' + String(page).padStart(4, '0'); +} + +/** + * The local date at `ms` as YYYY-MM-DD. `tzOffsetMin` is what + * Date#getTimezoneOffset gives where the scan ran (minutes behind UTC), so + * an evening scan in the US files under its own day. + */ +export function dayKeyOf(ms, tzOffsetMin = 0) { + return new Date(ms - tzOffsetMin * 60000).toISOString().slice(0, 10); +} + +export function expireAtMs(startMs) { + return startMs + PAGE_TTL_DAYS * 86400000; +} + +/** + * The cloud status for an extension run: active while it runs, then + * complete, stopped (by the user or a block page) or dead. + */ +export function runStatusOf(state) { + switch (state) { + case 'starting': + case 'running': + case 'stopping': + return 'active'; + case 'done': + return 'complete'; + case 'stopped': + case 'blocked': + return 'stopped'; + default: + return 'dead'; + } +} + +// ── Values ────────────────────────────────────────────────────── + +/** Milliseconds from a number, Date, ISO string or Timestamp; null otherwise. */ +export function timeMs(v) { + if (typeof v === 'number') return Number.isFinite(v) ? v : null; + if (v instanceof Date) return Number.isNaN(v.getTime()) ? null : v.getTime(); + if (typeof v === 'string') { + const t = Date.parse(v); + return Number.isNaN(t) ? null : t; + } + if (v && typeof v.toMillis === 'function') return v.toMillis(); + return null; +} + +const isInt = (v) => Number.isInteger(v); +const round1 = (n) => Math.round(n * 10) / 10; + +/** + * A compact point from an extension product record, leaving out what is + * unknown. + * @param {{priceCents?:?number, rating?:?number, reviewCount?:?number, isPrime?:boolean, organicRank?:?number}} rec + * @returns {Point} + */ +export function pointOf(rec) { + const pt = {}; + if (isInt(rec.priceCents) && rec.priceCents >= 0) pt.p = rec.priceCents; + if (typeof rec.rating === 'number' && rec.rating > 0 && rec.rating <= 5) pt.r = round1(rec.rating); + if (isInt(rec.reviewCount) && rec.reviewCount >= 0) pt.v = rec.reviewCount; + if (typeof rec.isPrime === 'boolean') pt.pr = rec.isPrime ? 1 : 0; + if (isInt(rec.organicRank) && rec.organicRank > 0) pt.rk = rec.organicRank; + return pt; +} + +/** + * The delta between two points, or null when nothing can be compared. + * @param {Point} now + * @param {?Point} prev + * @param {?number} [days] + * @returns {?Delta} + */ +export function deltaOf(now, prev, days = null) { + if (!prev) return null; + const d = {}; + if (isInt(now.p) && isInt(prev.p)) { + d.p = now.p - prev.p; + if (prev.p > 0) d.pPct = round1((d.p / prev.p) * 100); + } + if (typeof now.r === 'number' && typeof prev.r === 'number') d.r = round1(now.r - prev.r); + if (isInt(now.v) && isInt(prev.v)) d.v = now.v - prev.v; + if (Object.keys(d).length === 0) return null; + if (isInt(days) && days >= 0) d.days = days; + return d; +} + +// ── Validators ────────────────────────────────────────────────── +// Each returns a list of problems; empty means valid. + +const isStr = (v) => typeof v === 'string'; +const isStrOrNull = (v) => v === null || isStr(v); +const isTime = (v) => timeMs(v) !== null && typeof v !== 'string'; + +function checkPoint(pt, where, errs, { sp = false } = {}) { + if (!pt || typeof pt !== 'object') { errs.push(`${where}: not a map`); return; } + for (const [k, v] of Object.entries(pt)) { + if (v === null || v === undefined) errs.push(`${where}.${k}: unknown values are left out, not null`); + } + if ('p' in pt && !(isInt(pt.p) && pt.p >= 0)) errs.push(`${where}.p: cents must be a whole number`); + if ('r' in pt && !(typeof pt.r === 'number' && pt.r >= 0 && pt.r <= 5)) errs.push(`${where}.r: rating out of range`); + if ('v' in pt && !(isInt(pt.v) && pt.v >= 0)) errs.push(`${where}.v: review count must be a whole number`); + if ('pr' in pt && pt.pr !== 0 && pt.pr !== 1) errs.push(`${where}.pr: must be 0 or 1`); + if ('rk' in pt && !(isInt(pt.rk) && pt.rk > 0)) errs.push(`${where}.rk: rank must be 1 or more`); + if ('sp' in pt && (!sp || (pt.sp !== 0 && pt.sp !== 1))) errs.push(`${where}.sp: only page items carry sp, as 0 or 1`); +} + +function checkSv(doc, errs) { + if (!doc || typeof doc !== 'object') { errs.push('not a document'); return false; } + if (doc.sv !== SV) errs.push(`sv: expected ${SV}`); + return true; +} + +/** @returns {string[]} */ +export function validateSource(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.sourceId) || !/^[sk]_/.test(doc.sourceId)) errs.push('sourceId: expected s_ or k_'); + if (!SOURCE_TYPES.includes(doc.type)) errs.push('type: storefront or keyword'); + if (!isStrOrNull(doc.sellerId)) errs.push('sellerId: string or null'); + if (!isStrOrNull(doc.keyword)) errs.push('keyword: string or null'); + if (!isStrOrNull(doc.url)) errs.push('url: string or null'); + if (!isStr(doc.lastRunId)) errs.push('lastRunId: string'); + if (!isTime(doc.lastScrapedAt)) errs.push('lastScrapedAt: time'); + if ('catalogSize' in doc && !(isInt(doc.catalogSize) && doc.catalogSize >= 0)) errs.push('catalogSize: whole number'); + return errs; +} + +/** @returns {string[]} */ +export function validateRun(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isStr(doc.sourceId)) errs.push('sourceId: string'); + if (isStr(doc.runId) && isStr(doc.sourceId) && !doc.runId.startsWith(doc.sourceId + '_')) errs.push('runId: must start with sourceId_'); + if (!doc.source || !SOURCE_TYPES.includes(doc.source.type)) errs.push('source.type: storefront or keyword'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if (!isStr(doc.dayKey) || !DAY_KEY_RE.test(doc.dayKey)) errs.push('dayKey: YYYY-MM-DD'); + if (!isTime(doc.startedAt)) errs.push('startedAt: time'); + if (doc.finishedAt !== null && !isTime(doc.finishedAt)) errs.push('finishedAt: time or null'); + if (!RUN_STATUS.includes(doc.status)) errs.push(`status: one of ${RUN_STATUS.join(', ')}`); + if (doc.status === 'active' && doc.finishedAt !== null) errs.push('finishedAt: null while active'); + if (!isStrOrNull(doc.reason)) errs.push('reason: string or null'); + if (!isInt(doc.pagesDone) || doc.pagesDone < 0) errs.push('pagesDone: whole number'); + if (!isInt(doc.maxPages) || doc.maxPages < 1) errs.push('maxPages: whole number'); + if (doc.pagesPlanned !== null && !isInt(doc.pagesPlanned)) errs.push('pagesPlanned: whole number or null'); + if (doc.totalResultsOnSerp !== null && !isInt(doc.totalResultsOnSerp)) errs.push('totalResultsOnSerp: whole number or null'); + const c = doc.counters || {}; + for (const k of ['placements', 'uniqueAsins', 'sponsored', 'priceParseFailures', 'newSeen']) { + if (!isInt(c[k]) || c[k] < 0) errs.push(`counters.${k}: whole number`); + } + return errs; +} + +/** @returns {string[]} */ +export function validatePage(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.runId)) errs.push('runId: string'); + if (!isInt(doc.page) || doc.page < 1) errs.push('page: 1 or more'); + if (!isTime(doc.scrapedAt)) errs.push('scrapedAt: time'); + if (!isTime(doc.expireAt)) errs.push('expireAt: time'); + if (!isInt(doc.count) || doc.count < 0) errs.push('count: whole number'); + if (!isInt(doc.placements) || doc.placements < 0) errs.push('placements: whole number'); + if (!doc.items || typeof doc.items !== 'object') { + errs.push('items: map'); + } else { + const keys = Object.keys(doc.items); + if (keys.length > MAX_PAGE_ITEMS) errs.push(`items: at most ${MAX_PAGE_ITEMS}`); + for (const asin of keys) { + if (!ASIN_RE.test(asin)) errs.push(`items.${asin}: not an ASIN`); + checkPoint(doc.items[asin], `items.${asin}`, errs, { sp: true }); + } + } + return errs; +} + +/** @returns {string[]} */ +export function validateProduct(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (doc.mk !== MK) errs.push(`mk: ${MK}`); + if ('name' in doc && !isStrOrNull(doc.name)) errs.push('name: string or null'); + if ('img' in doc && doc.img !== null && !(isStr(doc.img) && /^https:\/\//.test(doc.img))) errs.push('img: https URL or null'); + if (!isStr(doc.url)) errs.push('url: string'); + if (!doc.latest) { + errs.push('latest: map'); + } else { + const { at, runId, dayKey, ...pt } = doc.latest; + checkPoint(pt, 'latest', errs); + if (!isTime(at)) errs.push('latest.at: time'); + if (!isStr(runId)) errs.push('latest.runId: string'); + if (!isStr(dayKey) || !DAY_KEY_RE.test(dayKey)) errs.push('latest.dayKey: YYYY-MM-DD'); + } + if (doc.prev !== null && doc.prev !== undefined) { + const { at, ...pt } = doc.prev; + checkPoint(pt, 'prev', errs); + if (at !== undefined && !isTime(at)) errs.push('prev.at: time'); + } + if (doc.delta !== null && doc.delta !== undefined) { + const d = doc.delta; + if ('p' in d && !isInt(d.p)) errs.push('delta.p: cents must be a whole number'); + if ('pPct' in d && typeof d.pPct !== 'number') errs.push('delta.pPct: number'); + if ('r' in d && typeof d.r !== 'number') errs.push('delta.r: number'); + if ('v' in d && !isInt(d.v)) errs.push('delta.v: whole number'); + if ('days' in d && !isInt(d.days)) errs.push('delta.days: whole number'); + for (const [k, v] of Object.entries(d)) if (v === null) errs.push(`delta.${k}: left out, not null`); + } + if ('sourceIds' in doc && !(Array.isArray(doc.sourceIds) && doc.sourceIds.every(isStr))) errs.push('sourceIds: list of strings'); + if ('firstSeenAt' in doc && !isTime(doc.firstSeenAt)) errs.push('firstSeenAt: time'); + return errs; +} + +/** @returns {string[]} */ +export function validateHistory(doc) { + const errs = []; + if (!checkSv(doc, errs)) return errs; + if (!isStr(doc.asin) || !ASIN_RE.test(doc.asin)) errs.push('asin: 10 letters and digits'); + if (!doc.d || typeof doc.d !== 'object') { + errs.push('d: map'); + } else { + for (const [day, pt] of Object.entries(doc.d)) { + if (!DAY_KEY_RE.test(day)) errs.push(`d.${day}: not a day key`); + checkPoint(pt, `d.${day}`, errs); + } + } + return errs; +} + +const VALIDATORS = { + source: validateSource, + run: validateRun, + page: validatePage, + product: validateProduct, + history: validateHistory +}; + +/** Throws when `doc` is not a valid document of `kind`. */ +export function assertValid(kind, doc) { + const errs = VALIDATORS[kind](doc); + if (errs.length) throw new Error(`invalid ${kind} document: ${errs.join('; ')}`); + return doc; +} diff --git a/scripts/check-schema.mjs b/scripts/check-schema.mjs new file mode 100644 index 0000000..e69d660 --- /dev/null +++ b/scripts/check-schema.mjs @@ -0,0 +1,28 @@ +// The vendored schema must be the extension's file byte for byte, and +// index.d.ts must declare everything index.js exports. +// +// node scripts/check-schema.mjs + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { WEB_ROOT, requireExtension } from './lib/extension.mjs'; + +const ours = resolve(WEB_ROOT, 'packages', 'schema', 'index.js'); +const types = readFileSync(resolve(WEB_ROOT, 'packages', 'schema', 'index.d.ts'), 'utf8'); +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` (${detail})`}`); +}; + +const src = readFileSync(ours, 'utf8'); +const exported = [...src.matchAll(/^export (?:const|function) (\w+)/gm)].map((m) => m[1]); +const missing = exported.filter((name) => !new RegExp(`declare (?:const|function) ${name}\\b`).test(types)); +check('index.d.ts declares every export', missing.length === 0, missing.join(', ')); + +const ext = requireExtension('the schema check'); +const theirs = readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8'); +check(`index.js matches ${ext}`, theirs === src, 'copy the extension file over ours'); + +console.log(failed === 0 ? 'RESULT: PASS' : `RESULT: FAIL (${failed})`); +process.exit(failed === 0 ? 0 : 1); diff --git a/scripts/lib/extension.mjs b/scripts/lib/extension.mjs new file mode 100644 index 0000000..08c368e --- /dev/null +++ b/scripts/lib/extension.mjs @@ -0,0 +1,27 @@ +// Finds the extension checkout next to this repo: PROSCAN_EXT, else +// ../ext or ../AmazonSellerScraper. The schema check, the sync rules test +// and the e2e seed all run the extension's own files from there. + +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; + +export const WEB_ROOT = resolve(import.meta.dirname, '..', '..'); + +export function findExtension(env = process.env) { + const candidates = env.PROSCAN_EXT + ? [resolve(env.PROSCAN_EXT)] + : [resolve(WEB_ROOT, '..', 'ext'), resolve(WEB_ROOT, '..', 'AmazonSellerScraper')]; + return candidates.find((dir) => existsSync(resolve(dir, 'packages', 'schema', 'index.js'))) ?? null; +} + +/** The checkout, or exit: these checks mean nothing without the real files. */ +export function requireExtension(what) { + const dir = findExtension(); + if (dir) return dir; + if (process.env.PROSCAN_ALLOW_NO_EXT === '1') { + console.log(`SKIP ${what}: no extension checkout (PROSCAN_ALLOW_NO_EXT=1)`); + process.exit(0); + } + console.error(`${what} needs the extension checkout. Set PROSCAN_EXT, or PROSCAN_ALLOW_NO_EXT=1 to skip.`); + process.exit(1); +} From 19043bcfbda8d1d6fa76f3b4cc166e05735d7b68 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:03:20 -0400 Subject: [PATCH 02/28] Type runs and pages from the shared schema --- .../src/features/drawer/HistoryDrawer.tsx | 2 +- dashboard/src/lib/types.ts | 57 ++++++++----------- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/dashboard/src/features/drawer/HistoryDrawer.tsx b/dashboard/src/features/drawer/HistoryDrawer.tsx index bb6afc4..8963b49 100644 --- a/dashboard/src/features/drawer/HistoryDrawer.tsx +++ b/dashboard/src/features/drawer/HistoryDrawer.tsx @@ -167,7 +167,7 @@ export default function HistoryDrawer({ wid, asin, onClose }: HistoryDrawerProps )}
-
+
{p.name ?? asin}
diff --git a/dashboard/src/lib/types.ts b/dashboard/src/lib/types.ts index dafccf7..f77c045 100644 --- a/dashboard/src/lib/types.ts +++ b/dashboard/src/lib/types.ts @@ -1,12 +1,15 @@ -// ProScan data contract — TypeScript mirrors of the Firestore schema in -// docs/architecture/data-model.md. All money fields are INTEGER CENTS; -// format exclusively via lib/format.ts. Compact point keys are shared by -// latest / prev / history.d values / page items. +// Dashboard types. The extension-written shapes come from the shared +// schema (packages/schema, vendored from the extension); this file adds +// the dashboard-owned fields (lead, tags, nickname, cadence) and the +// Phase 5 spread fields nothing writes yet. Money is integer cents; format +// it through lib/format.ts. import type { Timestamp } from 'firebase/firestore'; +import type { PageDoc as SchemaPageDoc, RunDoc } from '../../../packages/schema/index.js'; /** Compact observation point. Every field may be absent (e.g. `p` is - * omitted when the price parse failed). Money fields are cents. */ + * omitted when the price parse failed). Money fields are cents. p, r, v, + * pr, rk and sp are schema v1; the rest are reserved for Phase 5. */ export interface Point { /** price ¢ */ p?: number; @@ -134,14 +137,15 @@ export interface ProductLead { /** Canonical ASIN doc — workspaces/{wid}/products/{asin}. * spread / scores / verdict / delta / lead MAY BE ABSENT. */ export interface Product { + sv?: number; asin: string; mk: string; - name?: string; - img?: string; + name?: string | null; + img?: string | null; url?: string; latest?: ObservedPoint; - prev?: ObservedPoint; - delta?: ProductDelta; + prev?: ObservedPoint | null; + delta?: ProductDelta | null; spread?: ProductSpread; scores?: ProductScores; verdict?: ProductVerdict; @@ -162,42 +166,28 @@ export interface RunCounters { newSeen?: number; } -/** Scrape-run header — workspaces/{wid}/runs/{runId}. */ -export interface Run { - runId: string; - sourceId: string; - source?: { - type?: SourceType; - sellerId?: string | null; - keyword?: string | null; - url?: string; - }; - mk?: string; - /** UTC date of startedAt, 'YYYY-MM-DD' */ - dayKey?: string; - startedAt?: Timestamp; - /** null while active */ - finishedAt?: Timestamp | null; - status: RunStatus; - pagesDone?: number; - pagesPlanned?: number; - totalResultsOnSerp?: number; - counters?: RunCounters; +/** Scrape-run header — workspaces/{wid}/runs/{runId}. dayKey is the + * local date where the scan ran. */ +export type Run = RunDoc & { /** user-renamable run card */ label?: string | null; -} +}; + +/** One page of a run — workspaces/{wid}/runs/{runId}/pages/{pageId}. */ +export type PageDoc = SchemaPageDoc; export type SourceType = 'storefront' | 'keyword'; /** Watchlist entry — workspaces/{wid}/sources/{sourceId}. */ export interface Source { + sv?: number; sourceId: string; type: SourceType; /** null for keyword sources */ sellerId?: string | null; keyword?: string | null; nickname?: string | null; - url?: string; + url?: string | null; watched?: boolean; /** Rescan Queue: staleness = now − lastScrapedAt vs cadence */ cadenceDays?: number; @@ -210,8 +200,9 @@ export interface Source { } /** Date-keyed time-series — products/{asin}/history/daily. - * One Point per UTC dayKey; spread-day extras merged into the same entry. */ + * One Point per dayKey; spread-day extras merged into the same entry. */ export interface HistoryDoc { + sv?: number; asin: string; d: Record; } From 82303520c5e0f1d478758c2c7f6ace622ab19382 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:04:38 -0400 Subject: [PATCH 03/28] Check every document the dashboard reads against the schema --- dashboard/src/components/SchemaNotice.tsx | 27 +++++++ dashboard/src/components/components.css | 25 +++++++ .../src/features/drawer/HistoryDrawer.tsx | 5 ++ dashboard/src/features/drawer/data.ts | 13 +--- dashboard/src/lib/checked.ts | 73 +++++++++++++++++++ dashboard/src/lib/hooks.ts | 34 +++++---- dashboard/src/lib/queries.ts | 25 +++---- dashboard/src/routes/Products.tsx | 2 + dashboard/src/routes/Runs.tsx | 2 + dashboard/src/routes/Watchlist.tsx | 4 +- 10 files changed, 170 insertions(+), 40 deletions(-) create mode 100644 dashboard/src/components/SchemaNotice.tsx create mode 100644 dashboard/src/lib/checked.ts diff --git a/dashboard/src/components/SchemaNotice.tsx b/dashboard/src/components/SchemaNotice.tsx new file mode 100644 index 0000000..3f7e3da --- /dev/null +++ b/dashboard/src/components/SchemaNotice.tsx @@ -0,0 +1,27 @@ +import type { SchemaProblem } from '../lib/checked'; +import './components.css'; + +/** Says how many documents were left out for failing the schema check, + * with the first one's path and problems. Renders nothing when none were. */ +export default function SchemaNotice({ invalid }: { invalid: SchemaProblem[] }) { + if (invalid.length === 0) return null; + const first = invalid[0]; + const noun = invalid.length === 1 ? 'document' : 'documents'; + return ( +
+ + {invalid.length.toLocaleString('en-US')} {noun} left out + {' '} + because {invalid.length === 1 ? 'it does' : 'they do'} not match the ProScan schema. An + older extension build may have written {invalid.length === 1 ? 'it' : 'them'}. +
+ {first.path} +
    + {first.problems.slice(0, 6).map((p) => ( +
  • {p}
    • + ))} +
+
+
+ ); +} diff --git a/dashboard/src/components/components.css b/dashboard/src/components/components.css index 91c2cf0..1f37c42 100644 --- a/dashboard/src/components/components.css +++ b/dashboard/src/components/components.css @@ -446,3 +446,28 @@ line-height: 1; cursor: pointer; } + +/* ── notice: a thin banner above a view ─────────────────────────── */ +.notice { + margin: 0 0 var(--space-3); + padding: var(--space-2) var(--space-3); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + background: var(--surface); + color: var(--ink); + font-size: 13px; + line-height: 1.45; +} +.notice--warn { + border-color: var(--gold); + background: var(--gold-wash); +} +.notice__details { + margin-top: var(--space-1); + color: var(--muted); +} +.notice__details ul { + margin: var(--space-1) 0 0; + padding-left: var(--space-4); + font-size: 12px; +} diff --git a/dashboard/src/features/drawer/HistoryDrawer.tsx b/dashboard/src/features/drawer/HistoryDrawer.tsx index 8963b49..9d74063 100644 --- a/dashboard/src/features/drawer/HistoryDrawer.tsx +++ b/dashboard/src/features/drawer/HistoryDrawer.tsx @@ -12,6 +12,7 @@ import Drawer from '../../components/Drawer'; import EmptyState from '../../components/EmptyState'; import ErrorState from '../../components/ErrorState'; import { errorLabel } from '../../lib/errors'; +import SchemaNotice from '../../components/SchemaNotice'; import KeyValue from '../../components/KeyValue'; import Skeleton from '../../components/Skeleton'; import Sparkline, { type SparklinePoint } from '../../components/Sparkline'; @@ -149,6 +150,8 @@ export default function HistoryDrawer({ wid, asin, onClose }: HistoryDrawerProps ) : product.error ? ( + ) : product.invalid ? ( + ) : !p ? ( {history.loading ? ( + ) : history.invalid ? ( + ) : history.error ? (
Couldn't load history: {errorLabel(history.error)} diff --git a/dashboard/src/features/drawer/data.ts b/dashboard/src/features/drawer/data.ts index 04e5669..486b33e 100644 --- a/dashboard/src/features/drawer/data.ts +++ b/dashboard/src/features/drawer/data.ts @@ -9,22 +9,13 @@ // NEVER listeners on history / snapshots. import { useEffect, useMemo, useState } from 'react'; -import { - doc, - setDoc, - type DocumentData, - type FirestoreDataConverter, -} from 'firebase/firestore'; +import { doc, setDoc } from 'firebase/firestore'; import { db } from '../../firebase'; +import { productConverter } from '../../lib/checked'; import { useDocOnce, type DocOnceState } from '../../lib/hooks'; import { latestOfferSnapshot } from '../../lib/queries'; import type { OfferSnapshot, Product, ProductLead } from '../../lib/types'; -const productConverter: FirestoreDataConverter = { - toFirestore: (data) => data as DocumentData, - fromFirestore: (snap) => snap.data() as Product, -}; - /** One-shot read of workspaces/{wid}/products/{asin}. Re-fetches only when * the doc path changes (useDocOnce keys on path). */ export function useProductOnce(wid: string, asin: string): DocOnceState { diff --git a/dashboard/src/lib/checked.ts b/dashboard/src/lib/checked.ts new file mode 100644 index 0000000..5c43ed3 --- /dev/null +++ b/dashboard/src/lib/checked.ts @@ -0,0 +1,73 @@ +// Converters that check every document against the shared schema as it is +// read. A document that fails keeps its data but is marked, and the hooks +// leave it out of the view and count it, so a bad write shows up as a +// notice instead of a row of dashes. + +import type { + DocumentData, + FirestoreDataConverter, + QueryDocumentSnapshot, +} from 'firebase/firestore'; +import { + validateHistory, + validatePage, + validateProduct, + validateRun, + validateSource, +} from '../../../packages/schema/index.js'; +import type { HistoryDoc, PageDoc, Product, Run, Source } from './types'; + +export interface SchemaProblem { + path: string; + problems: string[]; +} + +const marks = new WeakMap(); + +/** The schema problems of a document read through a checked converter. */ +export function problemOf(value: object): SchemaProblem | undefined { + return marks.get(value); +} + +function checked( + validate: (doc: unknown) => string[], + // Documents the dashboard alone wrote (a lead note on a product the + // extension never saw, a nickname on a new source) carry no schema fields. + extensionWrote: (data: DocumentData) => boolean = () => true, +): FirestoreDataConverter { + return { + toFirestore: (data) => data as DocumentData, + fromFirestore(snap: QueryDocumentSnapshot) { + const data = snap.data(); + if (extensionWrote(data)) { + const problems = validate(data); + if (problems.length > 0) marks.set(data, { path: snap.ref.path, problems }); + } + return data as T; + }, + }; +} + +export const productConverter = checked( + validateProduct, + (d) => 'sv' in d || 'latest' in d, +); +export const runConverter = checked(validateRun); +export const pageConverter = checked(validatePage); +export const sourceConverter = checked(validateSource, (d) => 'sv' in d || 'lastRunId' in d); +export const historyConverter = checked(validateHistory, (d) => 'sv' in d); + +/** Splits rows into the ones that passed and the problems of the rest. */ +export function splitChecked(rows: T[]): { valid: T[]; invalid: SchemaProblem[] } { + const valid: T[] = []; + const invalid: SchemaProblem[] = []; + for (const row of rows) { + const problem = marks.get(row); + if (problem) invalid.push(problem); + else valid.push(row); + } + if (invalid.length > 0) { + console.warn(`[proscan] ${invalid.length} documents failed the schema check`, invalid.slice(0, 5)); + } + return { valid, invalid }; +} diff --git a/dashboard/src/lib/hooks.ts b/dashboard/src/lib/hooks.ts index 416ae5d..e2271a0 100644 --- a/dashboard/src/lib/hooks.ts +++ b/dashboard/src/lib/hooks.ts @@ -25,6 +25,7 @@ import { type Query, } from 'firebase/firestore'; import { auth, db } from '../firebase'; +import { problemOf, splitChecked, type SchemaProblem } from './checked'; import type { Workspace } from './types'; /* ── scan activity signal ───────────────────────────────────────────── @@ -88,7 +89,10 @@ export function useAuthUser(): AuthState { /* ── snapshot query (live, scoped, self-detaching) ──────────────────── */ export interface SnapshotQueryState { + /** documents that passed the schema check */ data: T[]; + /** documents left out because they failed it */ + invalid: SchemaProblem[]; loading: boolean; error: Error | null; } @@ -104,6 +108,7 @@ export function useSnapshotQuery( ): SnapshotQueryState { const [state, setState] = useState>({ data: [], + invalid: [], loading: true, error: null, }); @@ -111,7 +116,7 @@ export function useSnapshotQuery( useEffect(() => { const q = queryFactory(); if (!q) { - setState({ data: [], loading: false, error: null }); + setState({ data: [], invalid: [], loading: false, error: null }); return; } setState((s) => ({ ...s, loading: true, error: null })); @@ -132,11 +137,8 @@ export function useSnapshotQuery( `[proscan:reads] ${debugLabel ?? 'query'} +${delta} docs (mount total ${mountReads})`, ); } - setState({ - data: snap.docs.map((d) => d.data()), - loading: false, - error: null, - }); + const { valid, invalid } = splitChecked(snap.docs.map((d) => d.data() as T & object)); + setState({ data: valid, invalid, loading: false, error: null }); }, (error) => { if (firstSnapshot) { @@ -144,7 +146,7 @@ export function useSnapshotQuery( scanEnd(token); } console.error(`[proscan] ${debugLabel ?? 'query'} listener failed`, error); - setState({ data: [], loading: false, error }); + setState({ data: [], invalid: [], loading: false, error }); }, ); return () => { @@ -190,6 +192,8 @@ export function useServerCount( export interface DocOnceState { data: T | null; + /** set when the document exists but failed the schema check */ + invalid: SchemaProblem | null; loading: boolean; error: Error | null; } @@ -199,6 +203,7 @@ export interface DocOnceState { export function useDocOnce(ref: DocumentReference | null): DocOnceState { const [state, setState] = useState>({ data: null, + invalid: null, loading: ref !== null, error: null, }); @@ -206,26 +211,25 @@ export function useDocOnce(ref: DocumentReference | null): DocOnceState useEffect(() => { if (!ref) { - setState({ data: null, loading: false, error: null }); + setState({ data: null, invalid: null, loading: false, error: null }); return; } let cancelled = false; - setState({ data: null, loading: true, error: null }); + setState({ data: null, invalid: null, loading: true, error: null }); getDoc(ref) .then((snap) => { if (cancelled) return; if (import.meta.env.DEV) { console.debug(`[proscan:reads] doc-once ${snap.ref.path} +1`); } - setState({ - data: snap.exists() ? snap.data() : null, - loading: false, - error: null, - }); + const data = snap.exists() ? snap.data() : null; + const invalid = data && typeof data === 'object' ? (problemOf(data) ?? null) : null; + if (invalid) console.warn('[proscan] document failed the schema check', invalid); + setState({ data: invalid ? null : data, invalid, loading: false, error: null }); }) .catch((error: Error) => { console.error(`[proscan] read ${ref.path} failed`, error); - if (!cancelled) setState({ data: null, loading: false, error }); + if (!cancelled) setState({ data: null, invalid: null, loading: false, error }); }); return () => { cancelled = true; diff --git a/dashboard/src/lib/queries.ts b/dashboard/src/lib/queries.ts index 0afb005..369163c 100644 --- a/dashboard/src/lib/queries.ts +++ b/dashboard/src/lib/queries.ts @@ -23,6 +23,12 @@ import { type Query, } from 'firebase/firestore'; import { db } from '../firebase'; +import { + historyConverter, + productConverter, + runConverter, + sourceConverter, +} from './checked'; import type { HistoryDoc, LeadStage, @@ -34,20 +40,13 @@ import type { Source, } from './types'; -/* ── converters (id-stamping, cast-only — schema lives in types.ts) ─── */ - -function converter(): FirestoreDataConverter { - return { - toFirestore: (data) => data as DocumentData, - fromFirestore: (snap) => snap.data() as T, - }; -} +/* ── converters: checked against the shared schema (lib/checked.ts) ─── */ -const productConverter = converter(); -const runConverter = converter(); -const sourceConverter = converter(); -const historyConverter = converter(); -const snapshotConverter = converter(); +// Offer snapshots are Phase 5 and not in the schema yet. +const snapshotConverter: FirestoreDataConverter = { + toFirestore: (data) => data as DocumentData, + fromFirestore: (snap) => snap.data() as OfferSnapshot, +}; function productsCol(wid: string) { return collection(db, 'workspaces', wid, 'products').withConverter( diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index d0d96e1..ace508c 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -31,6 +31,7 @@ import type { Product } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Skeleton from '../components/Skeleton'; import Button from '../components/Button'; import HistoryDrawer from '../features/drawer/HistoryDrawer'; @@ -277,6 +278,7 @@ export default function Products() { /> } /> + {body} {asin && wid ? ( diff --git a/dashboard/src/routes/Runs.tsx b/dashboard/src/routes/Runs.tsx index 55866f6..606188b 100644 --- a/dashboard/src/routes/Runs.tsx +++ b/dashboard/src/routes/Runs.tsx @@ -12,6 +12,7 @@ import type { Source } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Button from '../components/Button'; import { CWS_URL } from '../auth/SignIn'; import RunCard, { RunCardSkeleton } from '../features/runs/RunCard'; @@ -97,6 +98,7 @@ export default function Runs() { title="Runs" subtitle="Your 30 most recent scrape runs — click one to open its products." /> + {body} ); diff --git a/dashboard/src/routes/Watchlist.tsx b/dashboard/src/routes/Watchlist.tsx index 54b01a6..01ee28a 100644 --- a/dashboard/src/routes/Watchlist.tsx +++ b/dashboard/src/routes/Watchlist.tsx @@ -9,6 +9,7 @@ import { staleness } from '../lib/format'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; import ErrorState from '../components/ErrorState'; +import SchemaNotice from '../components/SchemaNotice'; import Skeleton from '../components/Skeleton'; import Button from '../components/Button'; import { CWS_URL } from '../auth/SignIn'; @@ -21,7 +22,7 @@ function plural(n: number, word: string): string { export default function Watchlist() { const { wid } = useWorkspace(); - const { data, loading, error } = useSnapshotQuery( + const { data, invalid, loading, error } = useSnapshotQuery( () => (wid ? sources(wid) : null), [wid], 'watchlist:sources', @@ -74,6 +75,7 @@ export default function Watchlist() { subtitle="Storefronts and keywords on a rescan cadence — fresh within cadence, due past it, stale past twice it." actions={summary ? {summary} : undefined} /> + {body} ); From 0985ddb64431546bd691c9bcb24d1a04e86039f0 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:11:57 -0400 Subject: [PATCH 04/28] Build the rules tests' seed products from the shared schema --- scripts/lib/docs.mjs | 25 +++++++++++++++++++++++++ scripts/qa-board-count.mjs | 8 ++------ scripts/qa-lead-rules.mjs | 9 ++------- 3 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 scripts/lib/docs.mjs diff --git a/scripts/lib/docs.mjs b/scripts/lib/docs.mjs new file mode 100644 index 0000000..3f69de3 --- /dev/null +++ b/scripts/lib/docs.mjs @@ -0,0 +1,25 @@ +// Schema-valid documents for rules tests that need a product in place +// before they probe something else. Checked with the vendored schema, so a +// schema change breaks these loudly instead of quietly. + +import { Timestamp } from 'firebase/firestore'; +import { SV, MK, assertValid, dayKeyOf, runIdOf } from '../../packages/schema/index.js'; + +/** A product as sync writes it after one run of `sourceId`. */ +export function productDoc(asin, { cents = 1999, atMs = Date.parse('2026-09-01T12:00:00Z'), sourceId = 'k_qa' } = {}) { + const runId = runIdOf(sourceId, atMs); + const doc = { + sv: SV, + asin, + mk: MK, + name: `QA product ${asin}`, + url: `https://www.amazon.com/dp/${asin}`, + latest: { p: cents, r: 4.5, v: 120, at: Timestamp.fromMillis(atMs), runId, dayKey: dayKeyOf(atMs) }, + prev: null, + delta: null, + sourceIds: [sourceId], + firstSeenAt: Timestamp.fromMillis(atMs), + firstRunId: runId, + }; + return assertValid('product', doc); +} diff --git a/scripts/qa-board-count.mjs b/scripts/qa-board-count.mjs index b412d59..7057d82 100644 --- a/scripts/qa-board-count.mjs +++ b/scripts/qa-board-count.mjs @@ -16,8 +16,8 @@ import { query, where, writeBatch, - Timestamp, } from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'board-count'); const auth = getAuth(app); @@ -42,11 +42,7 @@ try { const batch = writeBatch(db); for (let i = 0; i < N; i++) { const asin = `B0QACNT${String(i).padStart(3, '0')}`; - batch.set(doc(col, asin), { - asin, - sourceIds: [i % 2 ? 's_ODD' : 's_EVEN'], - latest: { p: 1000 + i, dayKey: '2026-09-01', at: Timestamp.fromMillis(1e12 + i) }, - }); + batch.set(doc(col, asin), productDoc(asin, { cents: 1000 + i, atMs: 1e12 + i, sourceId: i % 2 ? 's_ODD' : 's_EVEN' })); } await batch.commit(); diff --git a/scripts/qa-lead-rules.mjs b/scripts/qa-lead-rules.mjs index 448b752..2d82f16 100644 --- a/scripts/qa-lead-rules.mjs +++ b/scripts/qa-lead-rules.mjs @@ -13,6 +13,7 @@ import { setDoc, serverTimestamp, } from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'lead-rules'); const auth = getAuth(app); @@ -48,13 +49,7 @@ try { const product = (asin) => doc(db, 'workspaces', uid, 'products', asin); // A product as sync.js writes it: no lead map at all. - const seed = (asin) => - setDoc(product(asin), { - asin, - mk: 'US', - name: 'QA lead product', - latest: { p: 1999, dayKey: '2026-09-01' }, - }); + const seed = (asin) => setDoc(product(asin), productDoc(asin)); console.log('[lead] notes on an untriaged product'); await seed('B0QALEAD01'); From 7088c9e92f43b41c384762792bea1025d33ee54c Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:11:58 -0400 Subject: [PATCH 05/28] Check every Firestore write against the shared schema, and test sync with the extension's own module Each document lists the keys it may carry and their types. ASIN ids must match the ASIN pattern, run ids must be {sourceId}_{startMs}, scan times may not be in the future, firstSeenAt and firstRunId are create-only, events and stage moves are stamped with request.time, and owners can delete their own documents. The sync rules test now runs the extension's engine and sync.js from its checkout instead of a hand-copied port. --- firestore.rules | 268 +++++++++++++++++---- scripts/lib/extension-sync.mjs | 159 +++++++++++++ scripts/qa-sync-rules.mjs | 412 ++++++++++----------------------- 3 files changed, 507 insertions(+), 332 deletions(-) create mode 100644 scripts/lib/extension-sync.mjs diff --git a/firestore.rules b/firestore.rules index a120a80..bbeee8c 100644 --- a/firestore.rules +++ b/firestore.rules @@ -1,4 +1,9 @@ rules_version = '2'; + +// Checked against the shared schema (packages/schema/index.js, SV 1). +// The extension writes sources, runs, pages, products and history; the +// dashboard writes users, workspaces, leads, tags, events and the watchlist +// fields on sources. Every document lists the keys it may carry. service cloud.firestore { match /databases/{db}/documents { function signedIn() { return request.auth != null; } @@ -7,77 +12,260 @@ service cloud.firestore { // Later (VA Seats): || request.auth.uid in get(/databases/$(db)/documents/workspaces/$(wid)).data.members } + function nat(v) { return v is int && v >= 0; } + function str(v, max) { return v is string && v.size() <= max; } + function strOrNull(v, max) { return v == null || str(v, max); } + function asinOk(a) { return a is string && a.matches('^[A-Z0-9]{10}$'); } + function dayKeyOk(k) { return k is string && k.matches('^[0-9]{4}-[0-9]{2}-[0-9]{2}$'); } + function sourceIdOk(id) { return id is string && id.matches('^[sk]_[A-Za-z0-9_-]{1,200}$'); } + + // Scan times come from the scanning machine's clock. An hour of slack + // lets a fast clock sync while still refusing a 2099 pin to the top. + function past(t) { return t is timestamp && t <= request.time + duration.value(1, 'h'); } + + // Set when the document is created, never changed after. + function createOnly(field) { + return resource == null + || request.resource.data.get(field, null) == resource.data.get(field, null); + } + + // A compact point (schema Point). `extra` names the provenance keys a + // point may also carry where it is stored. + function pointOk(pt, extra) { + return pt is map + && pt.keys().hasOnly(['p', 'r', 'v', 'pr', 'rk'].concat(extra)) + && (!('p' in pt) || nat(pt.p)) + && (!('r' in pt) || (pt.r is number && pt.r >= 0 && pt.r <= 5)) + && (!('v' in pt) || nat(pt.v)) + && (!('pr' in pt) || pt.pr in [0, 1]) + && (!('rk' in pt) || (pt.rk is int && pt.rk > 0)); + } + + function deltaOk(d) { + return d is map + && d.keys().hasOnly(['p', 'pPct', 'r', 'v', 'days']) + && (!('p' in d) || d.p is int) + && (!('pPct' in d) || d.pPct is number) + && (!('r' in d) || d.r is number) + && (!('v' in d) || d.v is int) + && (!('days' in d) || nat(d.days)); + } + match /users/{uid} { - allow read, write: if signedIn() && request.auth.uid == uid; + allow read, delete: if signedIn() && request.auth.uid == uid; + allow create, update: if signedIn() && request.auth.uid == uid + && request.resource.data.keys().hasOnly(['displayName', 'email', 'defaultWorkspace', 'createdAt']) + && strOrNull(request.resource.data.get('displayName', null), 200) + && strOrNull(request.resource.data.get('email', null), 320) + && request.resource.data.get('defaultWorkspace', uid) == uid + && (resource != null || request.resource.data.get('createdAt', request.time) == request.time) + && createOnly('createdAt'); } match /workspaces/{wid} { - allow read: if member(wid); - allow create: if member(wid) - && request.resource.data.ownerUid == request.auth.uid - && request.resource.data.get('plan', 'free') == 'free'; - allow update: if member(wid) - && request.resource.data.ownerUid == resource.data.ownerUid - && request.resource.data.get('plan', 'free') == resource.data.get('plan', 'free'); - allow delete: if false; + function workspaceOk() { + let d = request.resource.data; + return d.keys().hasOnly(['ownerUid', 'name', 'plan', 'settings', 'tagMeta', 'lastReviewedAt', 'createdAt']) + && d.ownerUid == wid + && strOrNull(d.get('name', null), 120) + && d.get('settings', {}) is map + && d.get('tagMeta', {}) is map + && (!('lastReviewedAt' in d) || past(d.lastReviewedAt)); + } + + allow read, delete: if member(wid); + allow create: if member(wid) && workspaceOk() + && request.resource.data.get('plan', 'free') == 'free' + && request.resource.data.get('createdAt', request.time) == request.time; + allow update: if member(wid) && workspaceOk() + && request.resource.data.get('plan', null) == resource.data.get('plan', null) + && createOnly('createdAt'); match /sources/{sourceId} { + function sourceOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'sourceId', 'type', 'sellerId', 'keyword', 'url', 'lastRunId', 'lastScrapedAt', 'catalogSize', + 'nickname', 'watched', 'cadenceDays', 'tags', 'notes', 'createdAt']) + && sourceIdOk(sourceId) + && d.get('sv', 1) == 1 + && d.get('sourceId', sourceId) == sourceId + && d.type in ['storefront', 'keyword'] + && strOrNull(d.get('sellerId', null), 40) + && strOrNull(d.get('keyword', null), 300) + && strOrNull(d.get('url', null), 2000) + && (!('lastRunId' in d) || str(d.lastRunId, 260)) + && (!('lastScrapedAt' in d) || past(d.lastScrapedAt)) + && (!('catalogSize' in d) || nat(d.catalogSize)) + && strOrNull(d.get('nickname', null), 120) + && d.get('watched', false) is bool + && (!('cadenceDays' in d) || (d.cadenceDays is int && d.cadenceDays >= 1 && d.cadenceDays <= 365)) + && (!('tags' in d) || (d.tags is list && d.tags.size() <= 30)) + && strOrNull(d.get('notes', null), 5000) + && createOnly('createdAt'); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.type in ['storefront', 'keyword'] - && request.resource.data.get('cadenceDays', 7) is int; + allow create, update: if member(wid) && sourceOk(); } match /runs/{runId} { + function runOk() { + let d = request.resource.data; + let c = d.counters; + return d.keys().hasOnly([ + 'sv', 'runId', 'sourceId', 'source', 'mk', 'dayKey', 'startedAt', 'finishedAt', 'status', 'reason', + 'pagesDone', 'maxPages', 'pagesPlanned', 'totalResultsOnSerp', 'counters', 'label']) + && d.sv == 1 + && sourceIdOk(d.sourceId) + && d.runId == runId + && d.startedAt is timestamp + && runId == d.sourceId + '_' + string(d.startedAt.toMillis()) + && past(d.startedAt) + && d.source is map + && d.source.keys().hasOnly(['type', 'sellerId', 'keyword', 'url']) + && d.source.type in ['storefront', 'keyword'] + && d.mk == 'US' + && dayKeyOk(d.dayKey) + && d.status in ['active', 'complete', 'stopped', 'dead'] + && (d.finishedAt == null || (past(d.finishedAt) && d.finishedAt >= d.startedAt)) + && (d.status != 'active' || d.finishedAt == null) + && strOrNull(d.reason, 100) + && nat(d.pagesDone) + && d.maxPages is int && d.maxPages >= 1 && d.maxPages <= 1000 + && (d.pagesPlanned == null || nat(d.pagesPlanned)) + && (d.totalResultsOnSerp == null || nat(d.totalResultsOnSerp)) + && c is map + && c.keys().hasOnly(['placements', 'uniqueAsins', 'sponsored', 'priceParseFailures', 'newSeen']) + && nat(c.placements) && nat(c.uniqueAsins) && nat(c.sponsored) + && nat(c.priceParseFailures) && nat(c.newSeen) + && strOrNull(d.get('label', null), 120); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.status in ['active', 'complete', 'stopped', 'dead'] - && request.resource.data.dayKey is string - && request.resource.data.sourceId is string; + allow create, update: if member(wid) && runOk(); match /pages/{pageId} { + function pageOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'runId', 'page', 'scrapedAt', 'expireAt', 'count', 'placements', 'kind', 'items', 'total', 'truncated']) + && d.sv == 1 + && d.runId == runId + && d.page is int && d.page >= 1 && d.page <= 1000 + && pageId.matches('^p[0-9]{4,}$') + && int(pageId[1:pageId.size()]) == d.page + && past(d.scrapedAt) + // TTL field: 400 days after the run started, so never further out than that. + && d.expireAt is timestamp + && d.expireAt <= request.time + duration.value(401, 'd') + && nat(d.count) + && nat(d.placements) + && str(d.kind, 20) + // Items can't be looped over here; the extension validates each. + && d.items is map + && d.items.size() <= 120 + && (!('total' in d) || nat(d.total)) + && (!('truncated' in d) || d.truncated is bool); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.items is map - && request.resource.data.items.size() <= 120 // Amazon pages carry ≤60 placements; 2× headroom - && request.resource.data.expireAt is timestamp; // TTL field is mandatory on cold docs + allow create, update: if member(wid) && pageOk(); } } match /products/{asin} { + function leadOk(lead) { + return lead is map + && lead.keys().hasOnly([ + 'stage', 'stageChangedAt', 'rejectedReason', 'notes', 'buyPrice', 'qty', 'supplier', 'orderRef', 'recheckAt']) + // A lead with no stored stage is 'new' (the drawer never writes it). + && lead.get('stage', 'new') in ['new', 'reviewing', 'approved', 'purchased', 'rejected', 'archived'] + // A stage move is stamped by the server, not the browser. + && (!('stageChangedAt' in lead) + || lead.stageChangedAt == request.time + || (resource != null + && resource.data.get('lead', {}).get('stageChangedAt', null) == lead.stageChangedAt)) + && strOrNull(lead.get('rejectedReason', null), 500) + && strOrNull(lead.get('notes', null), 5000) + && (lead.get('buyPrice', null) == null || nat(lead.buyPrice)) + && (lead.get('qty', null) == null || nat(lead.qty)) + && strOrNull(lead.get('supplier', null), 200) + && strOrNull(lead.get('orderRef', null), 200) + && (lead.get('recheckAt', null) == null || lead.recheckAt is timestamp); + } + + function productOk() { + let d = request.resource.data; + return d.keys().hasOnly([ + 'sv', 'asin', 'mk', 'name', 'url', 'img', 'latest', 'prev', 'delta', 'sourceIds', 'firstSeenAt', 'firstRunId', + 'lead', 'tags', 'verdict']) + && asinOk(asin) + && d.asin == asin // doc id == payload ASIN (dedup axis) + && d.get('sv', 1) == 1 + && d.get('mk', 'US') == 'US' + && strOrNull(d.get('name', null), 1000) + && (!('url' in d) || (str(d.url, 500) && d.url.matches('^https://www[.]amazon[.]com/.*'))) + && (d.get('img', null) == null || (str(d.img, 1000) && d.img.matches('^https://.*'))) + && (!('latest' in d) || ( + pointOk(d.latest, ['at', 'runId', 'dayKey']) + && past(d.latest.at) + && str(d.latest.runId, 260) + && dayKeyOk(d.latest.dayKey))) + && (d.get('prev', null) == null || ( + pointOk(d.prev, ['at']) + && (!('at' in d.prev) || past(d.prev.at)))) + && (d.get('delta', null) == null || deltaOk(d.delta)) + && (!('sourceIds' in d) || (d.sourceIds is list && d.sourceIds.size() <= 200)) + && (!('firstSeenAt' in d) || past(d.firstSeenAt)) + && (!('firstRunId' in d) || str(d.firstRunId, 260)) + && createOnly('firstSeenAt') + && createOnly('firstRunId') + && (!('lead' in d) || leadOk(d.lead)) + && (!('tags' in d) || (d.tags is list && d.tags.size() <= 30)) + && (!('verdict' in d) || (d.verdict is map && d.verdict.size() <= 8)); + } + allow read, delete: if member(wid); - allow create, update: if member(wid) - && request.resource.data.asin == asin // doc id == payload ASIN (dedup axis) - && (!('latest' in request.resource.data) || ( - request.resource.data.latest.dayKey is string - && (!('p' in request.resource.data.latest) || request.resource.data.latest.p is int) - && (!('v' in request.resource.data.latest) || request.resource.data.latest.v is int))) - // A lead with no stored stage is 'new' (the drawer never writes it). - && (!('lead' in request.resource.data) - || (request.resource.data.lead is map - && request.resource.data.lead.get('stage', 'new') in - ['new', 'reviewing', 'approved', 'purchased', 'rejected', 'archived'])); + allow create, update: if member(wid) && productOk(); match /history/{seriesId} { - allow read: if member(wid); - allow write: if member(wid) && seriesId in ['daily', 'monthly']; + allow read, delete: if member(wid); + allow create, update: if member(wid) + && seriesId in ['daily', 'monthly'] + && request.resource.data.keys().hasOnly(['sv', 'asin', 'd']) + && request.resource.data.sv == 1 + && request.resource.data.asin == asin + && request.resource.data.d is map + && request.resource.data.d.size() <= 1500; } match /offerSnapshots/{snapId} { allow read, delete: if member(wid); allow create, update: if member(wid) && request.resource.data.prices is list - && request.resource.data.expireAt is timestamp; + && request.resource.data.prices.size() <= 200 + && request.resource.data.expireAt is timestamp + && request.resource.data.expireAt <= request.time + duration.value(401, 'd'); } match /events/{eventId} { - allow read, create: if member(wid); - allow update, delete: if false; // append-only activity trail + allow read, delete: if member(wid); + // Append-only activity trail, stamped by the server. + allow create: if member(wid) + && request.resource.data.keys().hasOnly(['type', 'from', 'to', 'at', 'note']) + && str(request.resource.data.type, 40) + && strOrNull(request.resource.data.get('from', null), 40) + && strOrNull(request.resource.data.get('to', null), 40) + && strOrNull(request.resource.data.get('note', null), 2000) + && request.resource.data.at == request.time; + allow update: if false; } } - match /sellers/{sellerId} { allow read, write: if member(wid); } - match /rules/{ruleSetId} { allow read, write: if member(wid); } - match /views/{viewId} { allow read, write: if member(wid); } - match /alerts/{alertId} { allow read, write: if member(wid); } + // Nothing writes these yet. Owners can read and clear them. + match /sellers/{sellerId} { allow read, delete: if member(wid); } + match /rules/{ruleSetId} { allow read, delete: if member(wid); } + match /views/{viewId} { allow read, delete: if member(wid); } + match /alerts/{alertId} { allow read, delete: if member(wid); } } } } diff --git a/scripts/lib/extension-sync.mjs b/scripts/lib/extension-sync.mjs new file mode 100644 index 0000000..db5e0a2 --- /dev/null +++ b/scripts/lib/extension-sync.mjs @@ -0,0 +1,159 @@ +// Drives the extension's real run engine and sync module from its checkout: +// scraped pages go into the engine (IndexedDB via fake-indexeddb), and +// sync.js drains the outbox into Firestore under this repo's rules. Used by +// the sync rules test and the e2e seed, so neither writes hand-made data. +// +// sync.js takes its Firestore functions as a dependency, so it is handed +// this repo's firebase copy and writes through the caller's `db`. + +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + doc, + getDoc, + writeBatch, + Timestamp, + arrayUnion, + FieldPath, +} from 'firebase/firestore'; + +const FIRESTORE = { doc, getDoc, writeBatch, Timestamp, arrayUnion, FieldPath }; +const quiet = { log() {}, warn() {}, error() {} }; + +/** Loads the engine, db, sync and schema modules from an extension checkout. */ +export async function loadExtension(extDir) { + const require = createRequire(resolve(extDir, 'package.json')); + await import(pathToFileURL(require.resolve('fake-indexeddb/auto')).href); + const DB = require(resolve(extDir, 'scripts/background/db.js')); + const { createEngine } = require(resolve(extDir, 'scripts/background/engine.js')); + const { createSync } = await import(pathToFileURL(resolve(extDir, 'scripts/background/sync.js')).href); + return { DB, createEngine, createSync }; +} + +function memoryArea(init = {}) { + let data = JSON.parse(JSON.stringify(init)); + return { + async get(keys) { + const list = keys == null ? Object.keys(data) : [].concat(keys); + return Object.fromEntries(list.filter((k) => data[k] !== undefined).map((k) => [k, data[k]])); + }, + async set(items) { + Object.assign(data, JSON.parse(JSON.stringify(items))); + }, + async remove(keys) { + [].concat(keys).forEach((k) => delete data[k]); + }, + }; +} + +let installs = 0; + +/** + * One extension install signed in as `uid`, with a tab that answers every + * message. Returns scrape() to run a whole search and flush() to sync it. + */ +export function installExtension(ext, { uid, db, log = quiet }) { + const name = `proscan-web-${process.pid}-${++installs}`; + const clock = { t: 0 }; + let url = ''; + const chrome = { + storage: { + session: memoryArea(), + local: memoryArea({ account: { uid }, settings: { maxPages: 400 } }), + }, + runtime: { lastError: null }, + tabs: { + sendMessage(tabId, message, opts, cb) { + setImmediate(() => cb(message.type === 'PING' ? { ok: true, kind: 'results', url } : { ok: true })); + }, + async update() { + return {}; + }, + async get(id) { + return { id, url }; + }, + }, + }; + const engine = ext.createEngine({ + chrome, + openDb: () => ext.DB.open({ name }), + now: () => clock.t, + random: () => 0, + setTimer: () => 0, + clearTimer: () => {}, + flags: { CLOUD_SYNC: true }, + log, + }); + const sender = { tab: { id: 1 } }; + + async function reportPage(runId, pageNo, cards, { last, total }) { + if (pageNo > 1) { + clock.t += 5000; + await engine.tick(); + } + const at = new Date(clock.t).toISOString(); + const result = { + kind: last ? 'last' : 'results', + // Rank within the page; the engine makes it run-wide. + products: cards.map((p, j) => ({ + ...p, + placements: p.placements ?? [{ position: j + 1, sponsored: false, rank: j + 1 }], + scrapedAt: at, + })), + placements: cards.length, + nextHref: last ? null : `${url}&page=${pageNo + 1}`, + total, + fill: { asin: 1, title: 1, price: 1 }, + }; + const pageUrl = pageNo === 1 ? url : `${url}&page=${pageNo}`; + const resp = await engine.pageResult({ runId, page: pageNo, url: pageUrl, result }, sender); + if (!resp.ok) throw new Error(`page ${pageNo} refused: ${JSON.stringify(resp)}`); + } + + /** Scrapes `cards` from the search at `searchUrl`, `perPage` to a page, starting at `startMs`. */ + async function scrape(searchUrl, startMs, cards, perPage = 48) { + clock.t = startMs; + url = searchUrl; + const resp = await engine.start({ tabId: 1 }); + if (!resp.ok) throw new Error(`run refused: ${JSON.stringify(resp)}`); + const pages = Math.ceil(cards.length / perPage); + for (let i = 0; i < pages; i++) { + await reportPage(resp.runId, i + 1, cards.slice(i * perPage, (i + 1) * perPage), { + last: i === pages - 1, + total: cards.length, + }); + } + return resp.runId; + } + + const sync = ext.createSync({ db, openStore: () => engine.db(), fs: FIRESTORE, log }); + return { scrape, flush: () => sync.flush(uid), pending: () => sync.pending(uid) }; +} + +export const asinOf = (i) => `B0${i.toString(36).toUpperCase().padStart(8, '0')}`; + +/** + * Search result cards as the content script reports them. `price(i)` gives + * cents or null for a price that failed to parse. + */ +export function cards(n, { price = (i) => 1000 + i, reviews = (i) => 100 + i, offset = 0 } = {}) { + return Array.from({ length: n }, (_, k) => { + const i = k + offset; + const cents = price(i); + const asin = asinOf(i); + return { + asin, + name: `Test product ${i}`, + price: cents === null ? null : `$${(cents / 100).toFixed(2)}`, + priceCents: cents, + currency: cents === null ? null : 'USD', + rating: 3 + (i % 20) / 10, + reviewCount: reviews(i), + isPrime: i % 2 === 0, + sponsored: false, + url: `https://www.amazon.com/dp/${asin}`, + img: `https://m.media-amazon.com/images/I/test-${i}.jpg`, + }; + }); +} diff --git a/scripts/qa-sync-rules.mjs b/scripts/qa-sync-rules.mjs index 1eae650..93dcc46 100644 --- a/scripts/qa-sync-rules.mjs +++ b/scripts/qa-sync-rules.mjs @@ -1,33 +1,33 @@ -// Emulator integration QA — proves the ProScan extension's cloud-sync writes -// (scripts/background/sync.js :: syncToCloud) are ACCEPTED by the deployed -// firestore.rules, and that cross-tenant writes are DENIED. +// Sync rules test: the extension's real engine and sync.js (from its +// checkout, see scripts/lib/extension.mjs) write two runs of a storefront +// and a keyword into the emulator, and this repo's firestore.rules judge +// every write. Then every written document is read back and checked +// against the vendored schema, and the rules' create-only and tenant +// checks are probed directly. // -// Uses the firebase WEB SDK (NOT firebase-admin) so security rules are -// ENFORCED on every call — admin bypasses rules and would prove nothing. -// -// Connects ONLY to the local emulator suite. Hosts come from the variables -// `firebase emulators:exec` sets, defaulting to auth 9099 / firestore 8080. -// Run: npm run test:rules +// Web SDK, so rules are enforced. Run: npm run test:rules import { initializeApp, deleteApp } from 'firebase/app'; -import { - getAuth, - connectAuthEmulator, - createUserWithEmailAndPassword, - signInWithEmailAndPassword, -} from 'firebase/auth'; +import { getAuth, connectAuthEmulator, createUserWithEmailAndPassword, signOut } from 'firebase/auth'; import { getFirestore, connectFirestoreEmulator, - writeBatch, + collection, doc, getDoc, - serverTimestamp, + getDocs, + setDoc, + updateDoc, Timestamp, - arrayUnion, } from 'firebase/firestore'; +import * as Schema from '../packages/schema/index.js'; +import { requireExtension } from './lib/extension.mjs'; +import { asinOf, cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const extDir = requireExtension('the sync rules test'); +const ext = await loadExtension(extDir); -const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }); +const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, 'sync-rules'); const auth = getAuth(app); const db = getFirestore(app); const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST || '127.0.0.1:9099'; @@ -35,291 +35,119 @@ const [FS_HOST, FS_PORT] = (process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:80 connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); connectFirestoreEmulator(db, FS_HOST, Number(FS_PORT)); -const results = []; -const pass = (name, detail = '') => { results.push({ ok: true, name, detail }); console.log(` PASS ${name}${detail ? ' — ' + detail : ''}`); }; -const fail = (name, detail = '') => { results.push({ ok: false, name, detail }); console.log(` FAIL ${name}${detail ? ' — ' + detail : ''}`); }; -const check = (name, cond, detail = '') => (cond ? pass(name, detail) : fail(name, detail)); - -// ── helpers faithful to sync.js shapes ─────────────────────────────────────── -const round1 = (n) => Math.round(n * 10) / 10; -const intOrUndef = (n) => (typeof n === 'number' && Number.isFinite(n) ? Math.round(n) : undefined); -const tsOf = (iso) => Timestamp.fromDate(new Date(iso)); - -function pointFrom(q) { - const pt = { - p: intOrUndef(q.priceCents), - r: typeof q.rating === 'number' && q.rating > 0 ? round1(q.rating) : undefined, - v: typeof q.reviewCount === 'number' && q.reviewCount > 0 ? Math.round(q.reviewCount) : undefined, - pr: q.isPrime ? 1 : 0, - }; - Object.keys(pt).forEach((k) => pt[k] === undefined && delete pt[k]); - return pt; -} -function deltaBlock(q) { - const d = q.delta; - if (!d || d.isNew) return undefined; - const out = {}; - if (typeof d.dPriceCents === 'number') { - out.p = Math.round(d.dPriceCents); - const prev = typeof q.priceCents === 'number' ? q.priceCents - d.dPriceCents : null; - if (prev && prev !== 0) out.pPct = round1((d.dPriceCents / prev) * 100); - } - if (typeof d.dRating === 'number') out.r = round1(d.dRating); - if (typeof d.dReviews === 'number') out.v = Math.round(d.dReviews); - return Object.keys(out).length ? out : undefined; -} - -/** Faithful port of sync.js :: syncToCloud — same four write types/shapes. */ -async function syncToCloud(uid, bundle) { - const queue = (bundle && bundle.syncQueue) || []; - if (!queue.length) return { written: 0, runId: null, products: 0 }; - - const meta = (bundle && bundle.scrapeRunMeta) || {}; - const sourceId = meta.sellerId ? `s_${meta.sellerId}` : meta.keyword - ? `k_${String(meta.keyword).toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')}` - : 'k_unknown'; - const startMs = meta.startedAt ? Date.parse(meta.startedAt) : Date.now(); - const runId = `${sourceId}_${startMs}`; - const dayKey = new Date(startMs).toISOString().slice(0, 10); - const mk = 'US'; - - let products = 0; - const batch = writeBatch(db); - for (const q of queue) { - if (!q || !q.asin) continue; - const at = q.scrapedAt ? tsOf(q.scrapedAt) : serverTimestamp(); - const payload = { - asin: q.asin, - mk, - name: q.name, - url: q.url, - latest: { ...pointFrom(q), at, runId, dayKey }, - sourceIds: arrayUnion(sourceId), - }; - const d = deltaBlock(q); - if (d) payload.delta = d; - if (q.delta && q.delta.isNew) { - payload.firstSeenAt = at; - payload.firstRunId = runId; - } - // WRITE 1: products/{asin} - batch.set(doc(db, 'workspaces', uid, 'products', q.asin), payload, { merge: true }); - // WRITE 2: products/{asin}/history/daily - batch.set( - doc(db, 'workspaces', uid, 'products', q.asin, 'history', 'daily'), - { asin: q.asin, d: { [dayKey]: pointFrom(q) } }, - { merge: true }, - ); - products++; - } - await batch.commit(); - - const head = writeBatch(db); - // WRITE 3: runs/{runId} - head.set( - doc(db, 'workspaces', uid, 'runs', runId), - { - runId, - sourceId, - source: { type: meta.type || 'keyword', sellerId: meta.sellerId ?? null, keyword: meta.keyword ?? null, url: meta.url ?? null }, - mk, - dayKey, - startedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - finishedAt: serverTimestamp(), - status: 'complete', - pagesDone: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - pagesPlanned: bundle.scrapeRunPages ? bundle.scrapeRunPages.length : null, - counters: { - placements: queue.length, - uniqueAsins: new Set(queue.map((q) => q.asin)).size, - sponsored: 0, - priceParseFailures: queue.filter((q) => q.priceCents == null).length, - newSeen: queue.filter((q) => q.delta && q.delta.isNew).length, - }, - }, - { merge: true }, - ); - // WRITE 4: sources/{sourceId} - head.set( - doc(db, 'workspaces', uid, 'sources', sourceId), - { - sourceId, - type: meta.type === 'storefront' ? 'storefront' : 'keyword', - sellerId: meta.sellerId ?? null, - keyword: meta.keyword ?? null, - url: meta.url ?? null, - lastRunId: runId, - lastScrapedAt: meta.startedAt ? tsOf(meta.startedAt) : serverTimestamp(), - }, - { merge: true }, - ); - await head.commit(); - - return { written: products * 2 + 2, runId, products, sourceId, dayKey }; -} - -// ── sample producer queue: one NEW product + one RETURNING product ─────────── -const bundle = { - scrapeRunMeta: { - type: 'storefront', - sellerId: 'A3K9XELT4QZ6M2', - keyword: null, - url: 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2', - startedAt: '2026-06-21T10:00:00Z', - }, - scrapeRunPages: [{ page: 1 }, { page: 2 }], - syncQueue: [ - { - // RETURNING item — has a delta block (price dropped 200c from 2599 -> 2399) - asin: 'B0QASYNC01', - name: 'QA Returning Tumbler 40oz', - url: 'https://www.amazon.com/dp/B0QASYNC01', - priceCents: 2399, - rating: 4.6, - reviewCount: 1873, - isPrime: true, - scrapedAt: '2026-06-21T10:04:53Z', - delta: { isNew: false, dPriceCents: -200, dRating: 0.1, dReviews: 172 }, - }, - { - // NEW item — first-sight stamps, no delta - asin: 'B0QASYNC02', - name: 'QA New Mixing Bowl Set', - url: 'https://www.amazon.com/dp/B0QASYNC02', - priceCents: 1599, - rating: 4.1, - reviewCount: 233, - isPrime: false, - scrapedAt: '2026-06-21T10:05:10Z', - delta: { isNew: true }, - }, - ], +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` (${detail})`}`); }; - -let exitCode = 0; -try { - console.log('[auth] signing in test user A …'); - const emailA = `qa-sync-A-${Date.now()}@proscan.test`; - let credA; - try { - credA = await createUserWithEmailAndPassword(auth, emailA, 'qa-pass-123'); - } catch (e) { - credA = await signInWithEmailAndPassword(auth, emailA, 'qa-pass-123'); +// A bug that lives in the extension: reported, not fixed here. Strict, so +// the mark has to come off once the extension fixes it. +let known = 0; +const knownFailure = (name, ok, detail = '') => { + if (ok) { + failed++; + console.log(` FAIL ${name} now passes; remove its known-failure mark`); + } else { + known++; + console.log(` KNOWN FAILURE ${name}${detail ? ` (${detail})` : ''}`); } - const uidA = credA.user.uid; - console.log(` uid A = ${uidA}`); - - // ── POSITIVE: sync into the owner's own workspace ────────────────────────── - console.log('\n[positive] syncToCloud(uidA) — faithful sync.js writes'); - let res; +}; +async function denied(name, fn) { + let code = 'allowed'; try { - res = await syncToCloud(uidA, bundle); - pass('all 4 write types committed (rules ACCEPTED)', `products=${res.products}, runId=${res.runId}`); + await fn(); } catch (e) { - fail('writes rejected by rules', `${e.code || ''} ${e.message}`); - throw e; // can't read back if writes failed - } - - // ── READ-BACK each of the 4 write types ──────────────────────────────────── - console.log('\n[read-back] verifying each write type round-trips'); - - // 1 · products/{asin} — returning (B0QASYNC01) with delta - const p1 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC01')); - check('WRITE 1 products/{asin} (returning) exists', p1.exists()); - if (p1.exists()) { - const d = p1.data(); - check(' · asin field == doc id', d.asin === 'B0QASYNC01', `asin=${d.asin}`); - check(' · latest.dayKey is string "2026-06-21"', d.latest?.dayKey === '2026-06-21', `dayKey=${d.latest?.dayKey}`); - check(' · latest.p int == 2399', d.latest?.p === 2399, `p=${d.latest?.p}`); - check(' · latest.v int == 1873', d.latest?.v === 1873, `v=${d.latest?.v}`); - check(' · delta.p == -200, delta.pPct == -7.7', d.delta?.p === -200 && d.delta?.pPct === -7.7, `delta=${JSON.stringify(d.delta)}`); - check(' · sourceIds arrayUnion has s_A3K9XELT4QZ6M2', Array.isArray(d.sourceIds) && d.sourceIds.includes('s_A3K9XELT4QZ6M2'), JSON.stringify(d.sourceIds)); - check(' · returning item has NO firstSeenAt stamp', d.firstSeenAt === undefined); - } - - // 1b · products/{asin} — NEW (B0QASYNC02) with first-sight stamps - const p2 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC02')); - check('WRITE 1 products/{asin} (new) exists', p2.exists()); - if (p2.exists()) { - const d = p2.data(); - check(' · new item stamped firstRunId', d.firstRunId === res.runId, `firstRunId=${d.firstRunId}`); - check(' · new item has firstSeenAt timestamp', d.firstSeenAt instanceof Timestamp); - check(' · new item has NO delta block', d.delta === undefined); - check(' · latest.p int == 1599', d.latest?.p === 1599, `p=${d.latest?.p}`); - } - - // 2 · products/{asin}/history/daily - const h1 = await getDoc(doc(db, 'workspaces', uidA, 'products', 'B0QASYNC01', 'history', 'daily')); - check('WRITE 2 products/{asin}/history/daily exists', h1.exists()); - if (h1.exists()) { - const d = h1.data(); - check(' · history.d["2026-06-21"].p == 2399', d.d?.['2026-06-21']?.p === 2399, JSON.stringify(d.d?.['2026-06-21'])); + code = e.code || e.message; } + check(name, code === 'permission-denied', code); +} - // 3 · runs/{runId} - const r1 = await getDoc(doc(db, 'workspaces', uidA, 'runs', res.runId)); - check('WRITE 3 runs/{runId} exists', r1.exists()); - if (r1.exists()) { - const d = r1.data(); - check(' · status enum == "complete"', d.status === 'complete', `status=${d.status}`); - check(' · dayKey is string', typeof d.dayKey === 'string', `dayKey=${d.dayKey}`); - check(' · sourceId is string', typeof d.sourceId === 'string', `sourceId=${d.sourceId}`); - check(' · counters.placements == 2', d.counters?.placements === 2, `placements=${d.counters?.placements}`); - check(' · counters.newSeen == 1', d.counters?.newSeen === 1, `newSeen=${d.counters?.newSeen}`); - } +const DAY = 86400000; +const STORE = 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2&marketplaceID=ATVPDKIKX0DER'; +const KEYWORD = 'https://www.amazon.com/s?k=insulated+tumbler'; - // 4 · sources/{sourceId} - const s1 = await getDoc(doc(db, 'workspaces', uidA, 'sources', res.sourceId)); - check('WRITE 4 sources/{sourceId} exists', s1.exists()); - if (s1.exists()) { - const d = s1.data(); - check(' · type enum == "storefront"', d.type === 'storefront', `type=${d.type}`); - check(' · lastRunId == runId', d.lastRunId === res.runId, `lastRunId=${d.lastRunId}`); - check(' · cadenceDays omitted (rules default 7 is int)', d.cadenceDays === undefined); +try { + console.log(`[sync] extension at ${extDir}`); + const uid = (await createUserWithEmailAndPassword(auth, `qa-sync-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const ws = (...path) => doc(db, 'workspaces', uid, ...path); + const install = installExtension(ext, { uid, db }); + + const day1 = Date.parse('2026-06-02T15:00:00Z'); + const day8 = day1 + 7 * DAY; + const run1 = await install.scrape(STORE, day1, cards(130), 48); + // A week later every 10th price fails to parse and the rest drop by 100 cents. + const run2 = await install.scrape(STORE, day8, cards(130, { price: (i) => (i % 10 === 0 ? null : 900 + i) }), 48); + const run3 = await install.scrape(KEYWORD, day8 + 3600000, cards(20, { offset: 120 }), 48); + + console.log('[sync] flush through the rules'); + const totals = await install.flush(); + check('every outbox entry was written', totals.entries === 3 * 2 + 3 + 1 && (await install.pending()) === 0, JSON.stringify(totals)); + check('three runs, seven pages', totals.runs === 3 && totals.pages === 7, JSON.stringify(totals)); + + console.log('[sync] read back and check against the schema'); + const kinds = []; + for (const runId of [run1, run2, run3]) { + const run = (await getDoc(ws('runs', runId))).data(); + kinds.push(['run', runId, Schema.validateRun(run)]); + const pages = await getDocs(collection(db, 'workspaces', uid, 'runs', runId, 'pages')); + for (const p of pages.docs) kinds.push(['page', p.ref.path, Schema.validatePage(p.data())]); } - - // ── NEGATIVE: cross-tenant write must be DENIED ──────────────────────────── - console.log('\n[negative] cross-tenant write — uidA -> workspaces/{B}/products'); - const uidB = 'someOtherTenantUid_000000000000'; - let denied = false; - let errInfo = ''; - try { - await import('firebase/firestore').then(({ setDoc }) => - setDoc( - doc(db, 'workspaces', uidB, 'products', 'B0QASYNC01'), - { asin: 'B0QASYNC01', mk: 'US', name: 'cross-tenant intrusion', latest: { dayKey: '2026-06-21', p: 1 } }, - { merge: true }, - ), - ); - } catch (e) { - denied = e.code === 'permission-denied' || /permission/i.test(e.message); - errInfo = `${e.code || ''} ${e.message}`; + const products = await getDocs(collection(db, 'workspaces', uid, 'products')); + for (const p of products.docs) kinds.push(['product', p.id, Schema.validateProduct(p.data())]); + for (const id of ['s_A3K9XELT4QZ6M2', 'k_insulated-tumbler']) { + kinds.push(['source', id, Schema.validateSource((await getDoc(ws('sources', id))).data())]); } - check('cross-tenant write DENIED (permission-denied)', denied, denied ? errInfo : 'WRITE WAS ACCEPTED — RULES NOT ENFORCING TENANT ISOLATION'); - - // confirm nothing landed in B's workspace (read it back as A — also denied, - // so a thrown read is itself proof the doc is unreachable / not created) - let crossReadBlockedOrEmpty = false; + kinds.push(['history', asinOf(11), Schema.validateHistory((await getDoc(ws('products', asinOf(11), 'history', 'daily'))).data())]); + const bad = kinds.filter(([, , errs]) => errs.length > 0); + check(`${kinds.length} documents match schema v${Schema.SV}`, bad.length === 0, bad.slice(0, 3).map(([k, id, e]) => `${k} ${id}: ${e[0]}`).join('; ')); + check('140 products, one per ASIN', products.size === 140, `size=${products.size}`); + + const moved = (await getDoc(ws('products', asinOf(11)))).data(); + check('latest is the second run', moved.latest.runId === run2 && moved.latest.p === 911, JSON.stringify(moved.latest)); + check('delta against the first run', moved.delta?.p === -100 && moved.delta?.days === 7, JSON.stringify(moved.delta)); + check('firstSeenAt stays on the first run', moved.firstRunId === run1 && moved.firstSeenAt.toMillis() === day1); + const failedPrice = (await getDoc(ws('products', asinOf(10)))).data(); + check('a price that failed to parse is not kept from last week', !('p' in failedPrice.latest)); + const both = (await getDoc(ws('products', asinOf(125)))).data(); + // sync-plan.js names sourceIds in mergeFields, and a field in the mask is + // replaced, so arrayUnion keeps only the last source. + knownFailure('NEW-SYNC-1 (F-20 area): a product in two sources lists both', both.sourceIds.length === 2, JSON.stringify(both.sourceIds)); + check('rank is written', Number.isInteger(moved.latest.rk), JSON.stringify(moved.latest)); + + const header = (await getDoc(ws('runs', run2))).data(); + check('run header counts the run', header.status === 'complete' && header.counters.uniqueAsins === 130 && header.pagesDone === 3, JSON.stringify(header.counters)); + + console.log('[rules] create-only and forged values'); + await denied('rewrite firstSeenAt', () => updateDoc(ws('products', asinOf(11)), { firstSeenAt: Timestamp.now() })); + await denied('drop firstRunId', () => setDoc(ws('products', asinOf(11)), { ...moved, firstRunId: null })); + await denied('latest.at in 2099', () => updateDoc(ws('products', asinOf(11)), { 'latest.at': Timestamp.fromDate(new Date('2099-01-01')) })); + await denied('price as a string', () => updateDoc(ws('products', asinOf(11)), { 'latest.p': '9.11' })); + await denied('unknown product field', () => updateDoc(ws('products', asinOf(11)), { spread: { sc: 3 } })); + await denied('run id that is not {sourceId}_{startMs}', () => setDoc(ws('runs', 'k_x_1'), { ...header, runId: 'k_x_1' })); + await denied('page chunk under the wrong run', () => setDoc(ws('runs', run1, 'pages', 'p0009'), { sv: 1, runId: run2, page: 9, scrapedAt: Timestamp.now(), expireAt: Timestamp.now(), count: 0, placements: 0, kind: 'results', items: {} })); + + console.log('[rules] another account'); + await signOut(auth); + await createUserWithEmailAndPassword(auth, `qa-sync-b-${Date.now()}@proscan.test`, 'qa-pass-123'); + const intruder = installExtension(ext, { uid, db }); + await intruder.scrape(KEYWORD, day8 + 7200000, cards(3), 48); + let code = 'allowed'; try { - const bdoc = await getDoc(doc(db, 'workspaces', uidB, 'products', 'B0QASYNC01')); - crossReadBlockedOrEmpty = !bdoc.exists(); + await intruder.flush(); } catch (e) { - crossReadBlockedOrEmpty = true; // read denied too — also fine + code = e.code; } - check('cross-tenant doc absent/unreadable for uidA', crossReadBlockedOrEmpty); + check('sync into another workspace is refused', code === 'permission-denied', code); + check('and its entries stay queued', (await intruder.pending()) === 2); + await denied('reading another workspace', () => getDoc(ws('products', asinOf(11)))); } catch (e) { - console.error('\n[fatal]', e); - exitCode = 1; + console.error('[fatal]', e); + failed++; } finally { - // ── summary ──────────────────────────────────────────────────────────────── - const failed = results.filter((r) => !r.ok); - console.log('\n──────── SUMMARY ────────'); - console.log(`assertions: ${results.length} passed: ${results.length - failed.length} failed: ${failed.length}`); - if (failed.length) { - for (const f of failed) console.log(` FAILED: ${f.name} ${f.detail}`); - exitCode = 1; - } - console.log(exitCode === 0 ? '\nRESULT: PASS' : '\nRESULT: FAIL'); - try { await deleteApp(app); } catch {} - process.exit(exitCode); + if (known) console.log(`\n${known} known failure(s), see above`); + console.log(failed === 0 ? '\nRESULT: PASS' : `\nRESULT: FAIL (${failed})`); + try { + await deleteApp(app); + } catch {} + process.exit(failed === 0 ? 0 : 1); } From 399f3c7b1dbf40fc1de5681dd5ca93033187cbca Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:11:58 -0400 Subject: [PATCH 06/28] Keep the audit's adversarial rules probes as a permanent test --- package.json | 2 +- scripts/qa-adversarial-rules.mjs | 152 +++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 scripts/qa-adversarial-rules.mjs diff --git a/package.json b/package.json index 7020719..f2689e2 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "emulators": "node scripts/emulators.mjs start", "emulators:persist": "node scripts/emulators.mjs start --import=./.emulator-data --export-on-exit=./.emulator-data", "seed": "node scripts/seed-emulators.mjs", - "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs\"", + "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", "check:schema": "node scripts/check-schema.mjs", diff --git a/scripts/qa-adversarial-rules.mjs b/scripts/qa-adversarial-rules.mjs new file mode 100644 index 0000000..1fa4df5 --- /dev/null +++ b/scripts/qa-adversarial-rules.mjs @@ -0,0 +1,152 @@ +// Adversarial rules probes, from the audit's firestore-rules harness: tenant +// isolation, plan and owner forgery, oversized and malformed documents, +// forged timestamps, the append-only trail, owner delete, and the writes +// the dashboard really makes. Two signed-in tenants, web SDK, so rules are +// enforced. Run: npm run test:rules + +import { initializeApp, deleteApp } from 'firebase/app'; +import { getAuth, connectAuthEmulator, createUserWithEmailAndPassword } from 'firebase/auth'; +import { + getFirestore, + connectFirestoreEmulator, + collection, + deleteDoc, + deleteField, + doc, + getDoc, + getDocs, + serverTimestamp, + setDoc, + updateDoc, + Timestamp, +} from 'firebase/firestore'; +import { productDoc } from './lib/docs.mjs'; + +const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST || '127.0.0.1:9099'; +const [FS_HOST, FS_PORT] = (process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:8080').split(':'); + +const apps = []; +function tenant(name) { + const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, name); + apps.push(app); + const auth = getAuth(app); + const db = getFirestore(app); + connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); + connectFirestoreEmulator(db, FS_HOST, Number(FS_PORT)); + return { auth, db }; +} + +let failed = 0; +let count = 0; +async function probe(name, want, fn) { + count++; + let got = 'allow'; + let detail = ''; + try { + await fn(); + } catch (e) { + got = e.code === 'permission-denied' ? 'deny' : 'error'; + detail = ` (${e.code || e.message})`; + } + const ok = got === want; + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}: want ${want}, got ${got}${ok ? '' : detail}`); +} + +const big = 'x'.repeat(900 * 1024); +const future = Timestamp.fromDate(new Date('2099-01-01')); +const past = Timestamp.fromDate(new Date('2001-01-01')); + +try { + const A = tenant('adv-a'); + const B = tenant('adv-b'); + const ua = (await createUserWithEmailAndPassword(A.auth, `adv-a-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const ub = (await createUserWithEmailAndPassword(B.auth, `adv-b-${Date.now()}@proscan.test`, 'qa-pass-123')).user.uid; + const a = A.db; + const b = B.db; + + console.log('[bootstrap] the dashboard sign-in writes'); + await probe('users/{uid} profile mirror', 'allow', () => + setDoc(doc(b, 'users', ub), { displayName: 'B', email: 'b@proscan.test', defaultWorkspace: ub, createdAt: serverTimestamp() }, { merge: true }), + ); + await probe('workspace create', 'allow', () => + setDoc(doc(b, 'workspaces', ub), { ownerUid: ub, name: "B's workspace", plan: 'free', settings: { defaultRoiPct: 30 }, createdAt: serverTimestamp() }, { merge: true }), + ); + await setDoc(doc(a, 'workspaces', ua), { ownerUid: ua, plan: 'free', createdAt: serverTimestamp() }); + await setDoc(doc(b, 'workspaces', ub, 'products', 'B0SECRET01'), productDoc('B0SECRET01')); + + console.log('[tenants] isolation'); + await probe('A reads users/B', 'deny', () => getDoc(doc(a, 'users', ub))); + await probe('A reads workspaces/B', 'deny', () => getDoc(doc(a, 'workspaces', ub))); + await probe('A reads a B product', 'deny', () => getDoc(doc(a, 'workspaces', ub, 'products', 'B0SECRET01'))); + await probe('A lists B products', 'deny', () => getDocs(collection(a, 'workspaces', ub, 'products'))); + await probe('A writes B sellers', 'deny', () => setDoc(doc(a, 'workspaces', ub, 'sellers', 'x'), { x: 1 })); + await probe('A squats an unclaimed workspace', 'deny', () => setDoc(doc(a, 'workspaces', 'unclaimedUid123'), { ownerUid: ua, plan: 'free' })); + await probe('A deletes a B product', 'deny', () => deleteDoc(doc(a, 'workspaces', ub, 'products', 'B0SECRET01'))); + await probe('A deletes workspaces/B', 'deny', () => deleteDoc(doc(a, 'workspaces', ub))); + + console.log('[forgery] plan and owner'); + await probe('A sets its plan to pro', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { plan: 'pro' })); + await probe('A changes ownerUid', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { ownerUid: ub })); + await probe('A writes plan and role on users/A', 'deny', () => setDoc(doc(a, 'users', ua), { plan: 'pro', role: 'admin' }, { merge: true })); + await probe('A adds members and seats', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { members: [ub], seats: 999 })); + await probe('A writes a limit counter', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'counters', 'sources'), { count: 0 })); + await probe('A removes its plan field', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { plan: deleteField() })); + await probe('A rewrites workspace createdAt', 'deny', () => updateDoc(doc(a, 'workspaces', ua), { createdAt: past })); + + console.log('[shape] oversized and malformed'); + await probe('900 KB product name', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0BIG00001'), { ...productDoc('B0BIG00001'), name: big })); + await probe('product with junk fields and a bad id', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'NOTANASIN!'), { asin: 'NOTANASIN!', junk: { deep: [1, 2, 3] }, spread: 'lol' })); + await probe('negative price and a bad dayKey', 'deny', () => { + const d = productDoc('B0NEG00001'); + return setDoc(doc(a, 'workspaces', ua, 'products', 'B0NEG00001'), { ...d, latest: { ...d.latest, p: -500, dayKey: 'not-a-date' } }); + }); + await probe('1400-byte product id', 'deny', () => { + const id = 'Z'.repeat(1400); + return setDoc(doc(a, 'workspaces', ua, 'products', id), { asin: id }); + }); + await probe('javascript: product url', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0URL00001'), { ...productDoc('B0URL00001'), url: 'javascript:alert(1)' })); + await probe('history d as a 900 KB string', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOEXIST1', 'history', 'daily'), { sv: 1, asin: 'B0NOEXIST1', d: big })); + await probe('event with a 900 KB payload', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOEXIST1', 'events', 'e1'), { type: 'x', blob: big, at: serverTimestamp() })); + await probe('900 KB sellers doc', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sellers', 'S1'), { blob: big })); + await probe('arbitrary alert webhook', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'alerts', 'a1'), { webhook: 'http://evil' })); + await probe('run with forged counters', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'runs', 'r1'), { status: 'complete', dayKey: 'x', sourceId: 'y', counters: { uniqueAsins: -99999, placements: 'many' } })); + await probe('source cadenceDays 0', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_s1'), { type: 'keyword', cadenceDays: 0 })); + await probe('page chunk expiring in 9999', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'runs', 'k_x_1', 'pages', 'p0001'), { sv: 1, runId: 'k_x_1', page: 1, scrapedAt: Timestamp.now(), expireAt: Timestamp.fromDate(new Date('9999-01-01')), count: 0, placements: 0, kind: 'results', items: {} })); + + console.log('[time] forged timestamps'); + await probe('latest.at in 2099', 'deny', () => { + const d = productDoc('B0FUTURE01'); + return setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), { ...d, latest: { ...d.latest, at: future } }); + }); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), productDoc('B0FUTURE01')); + await probe('event backdated to 2001', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e2'), { type: 'stageChange', from: 'new', to: 'purchased', at: past })); + await probe('lead.stageChangedAt forged', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'), { lead: { stage: 'approved', stageChangedAt: future } }, { merge: true })); + + console.log('[events] append-only, owner may delete'); + await probe('event stamped by the server', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'), { type: 'stageChange', from: 'new', to: 'reviewing', at: serverTimestamp() })); + await probe('event overwrite', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'), { type: 'rewritten', at: serverTimestamp() })); + await probe('owner deletes an event', 'allow', () => deleteDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01', 'events', 'e3'))); + await probe('owner deletes a product', 'allow', () => deleteDoc(doc(a, 'workspaces', ua, 'products', 'B0FUTURE01'))); + + console.log('[dashboard] real write shapes'); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), productDoc('B0NOLEAD01')); + await probe('notes on a product with no stage yet', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { notes: 'call supplier' } }, { merge: true })); + await probe('stage move then notes', 'allow', async () => { + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { stage: 'reviewing', stageChangedAt: serverTimestamp() } }, { merge: true }); + await setDoc(doc(a, 'workspaces', ua, 'products', 'B0NOLEAD01'), { lead: { notes: 'second look' } }, { merge: true }); + }); + await probe('stage move on an ASIN with no product', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'products', 'B0GHOST001'), { lead: { stage: 'reviewing', stageChangedAt: serverTimestamp() } }, { merge: true })); + await setDoc(doc(a, 'workspaces', ua, 'sources', 'k_known'), { sv: 1, sourceId: 'k_known', type: 'keyword', sellerId: null, keyword: 'known', url: null, lastRunId: 'k_known_1', lastScrapedAt: Timestamp.now() }); + await probe('nickname on a scanned source', 'allow', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_known'), { nickname: 'Known' }, { merge: true })); + await probe('nickname on a source never scanned', 'deny', () => setDoc(doc(a, 'workspaces', ua, 'sources', 'k_new'), { nickname: 'x' }, { merge: true })); + await probe('owner deletes its workspace', 'allow', () => deleteDoc(doc(a, 'workspaces', ua))); + await probe('owner deletes its user profile', 'allow', () => deleteDoc(doc(b, 'users', ub))); +} catch (e) { + console.error('[fatal]', e); + failed++; +} finally { + console.log(failed === 0 ? `\n${count} probes. RESULT: PASS` : `\n${count} probes. RESULT: FAIL (${failed})`); + for (const app of apps) await deleteApp(app).catch(() => {}); + process.exit(failed === 0 ? 0 : 1); +} From 78b50251563625da95601c40276da2dcb80cd075 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:13:50 -0400 Subject: [PATCH 07/28] Page the product board with cursors and always show the true count The first 200 products are live and Load more pages on from the last loaded document, so the header never presents a window as the whole workspace (F-45). An exact ASIN in the search box is read directly. --- dashboard/src/features/board/board.css | 12 +++ dashboard/src/lib/paging.ts | 130 +++++++++++++++++++++++ dashboard/src/lib/queries.ts | 5 + dashboard/src/routes/Products.tsx | 141 +++++++++++++++---------- 4 files changed, 230 insertions(+), 58 deletions(-) create mode 100644 dashboard/src/lib/paging.ts diff --git a/dashboard/src/features/board/board.css b/dashboard/src/features/board/board.css index 311d9eb..b0b5fcb 100644 --- a/dashboard/src/features/board/board.css +++ b/dashboard/src/features/board/board.css @@ -303,3 +303,15 @@ .board-legend__note { color: var(--muted); } + +/* ── load more ─────────────────────────────────────────────────── */ +.board-more { + display: flex; + align-items: center; + gap: var(--space-3); + margin-top: var(--space-3); +} +.board-more__count { + color: var(--muted); + font-size: 12px; +} diff --git a/dashboard/src/lib/paging.ts b/dashboard/src/lib/paging.ts new file mode 100644 index 0000000..348000b --- /dev/null +++ b/dashboard/src/lib/paging.ts @@ -0,0 +1,130 @@ +// Cursor pagination over an ordered query. The first page is a live +// listener, so a scan that is syncing shows up without a reload; each +// "load more" is a one-shot getDocs that starts after the last document +// already loaded. Nothing is ever capped silently: `hasMore` says when +// the server has more. + +import { useCallback, useEffect, useMemo, useRef, useState, type DependencyList } from 'react'; +import { + getDocs, + limit, + onSnapshot, + query, + startAfter, + type Query, + type QueryDocumentSnapshot, +} from 'firebase/firestore'; +import { splitChecked, type SchemaProblem } from './checked'; +import { reportError } from './errors'; + +export const PAGE_SIZE = 200; + +export interface PagedState { + data: T[]; + invalid: SchemaProblem[]; + loading: boolean; + error: Error | null; + hasMore: boolean; + loadingMore: boolean; + loadMore: () => void; +} + +interface Page { + rows: T[]; + invalid: SchemaProblem[]; + last: QueryDocumentSnapshot | null; + full: boolean; +} + +function toPage(docs: QueryDocumentSnapshot[], size: number): Page { + const { valid, invalid } = splitChecked(docs.map((d) => d.data() as T & object)); + return { rows: valid, invalid, last: docs[docs.length - 1] ?? null, full: docs.length >= size }; +} + +/** + * Pages through `queryFactory()` (ordered, no limit) `size` documents at a + * time. `keyOf` dedupes a row that moved from one page into another while + * the live first page changed. + */ +export function usePagedQuery( + queryFactory: () => Query | null, + deps: DependencyList, + keyOf: (row: T) => string, + size: number = PAGE_SIZE, +): PagedState { + const [first, setFirst] = useState | null>(null); + const [more, setMore] = useState[]>([]); + const [error, setError] = useState(null); + const [loadingMore, setLoadingMore] = useState(false); + const [active, setActive] = useState(true); + const base = useRef | null>(null); + const generation = useRef(0); + + useEffect(() => { + const q = queryFactory(); + base.current = q; + generation.current++; + setFirst(null); + setMore([]); + setError(null); + setLoadingMore(false); + setActive(q !== null); + if (!q) return; + return onSnapshot( + query(q, limit(size)), + (snap) => setFirst(toPage(snap.docs, size)), + (err) => { + console.error('[proscan] first page failed', err); + setError(err); + }, + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps); + + const lastPage = more.length > 0 ? more[more.length - 1] : first; + const hasMore = !!lastPage && lastPage.full; + + const loadMore = useCallback(() => { + const q = base.current; + if (!q || !lastPage?.last || loadingMore) return; + const gen = generation.current; + setLoadingMore(true); + getDocs(query(q, startAfter(lastPage.last), limit(size))) + .then((snap) => { + if (gen !== generation.current) return; + setMore((pages) => [...pages, toPage(snap.docs, size)]); + }) + .catch((err: unknown) => { + if (gen === generation.current) reportError('load more products', err); + }) + .finally(() => { + if (gen === generation.current) setLoadingMore(false); + }); + }, [lastPage, loadingMore, size]); + + const { data, invalid } = useMemo(() => { + const pages = first ? [first, ...more] : []; + const seen = new Set(); + const rows: T[] = []; + for (const page of pages) { + for (const row of page.rows) { + const k = keyOf(row); + if (seen.has(k)) continue; + seen.add(k); + rows.push(row); + } + } + return { data: rows, invalid: pages.flatMap((p) => p.invalid) }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [first, more]); + + return { + data, + invalid, + loading: !error && active && first === null, + error, + hasMore, + loadingMore, + loadMore, + }; +} diff --git a/dashboard/src/lib/queries.ts b/dashboard/src/lib/queries.ts index 369163c..768e198 100644 --- a/dashboard/src/lib/queries.ts +++ b/dashboard/src/lib/queries.ts @@ -87,6 +87,11 @@ export function recentProducts(wid: string, lim: number | null = 200): Query { + return doc(db, 'workspaces', wid, 'products', asin).withConverter(productConverter); +} + /* ── runs / sources ─────────────────────────────────────────────────── */ /** Run Inbox — newest first, HARD-capped at 30 (read hygiene). */ diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index ace508c..f4c6281 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -2,15 +2,14 @@ // Excel-killer. URL contract (FROZEN): ?source= scopes the board // to one source; ?asin= opens the HistoryDrawer. // -// Read hygiene (docs/ops/billing-runbook.md): exactly ONE scoped products -// listener at a time — -// movers -> topMovers(wid, 100) (global biggest drops) -// source -> productsBySource(wid, sourceId, 500) -// default -> recentProducts(wid, 300) -// plus the sanctioned tiny sources listener for the scope dropdown. -// Search, sorting, export and the movers' has-delta filter are client-side -// over the loaded set only, so the header says when that set is capped and -// shows the server-side total (F-45). Real pagination lands in Phase 4. +// Reads (docs/ops/billing-runbook.md): +// latest -> productsBySource / recentProducts, 200 at a time: the first +// page is live, "Load more" pages on with a cursor (F-45) +// total -> getCountFromServer on the same query, so the header always +// says how many there are, not how many are loaded +// movers -> topMovers(wid, 100) +// plus the sanctioned tiny sources listener for the scope dropdown. An +// exact ASIN in the search box is looked up directly, loaded or not. import { useMemo, useState, type ReactNode } from 'react'; import { useSearchParams } from 'react-router-dom'; @@ -20,13 +19,16 @@ import { useReactTable, type SortingState, } from '@tanstack/react-table'; -import { useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { useDocOnce, useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { usePagedQuery, PAGE_SIZE } from '../lib/paging'; import { + productRef, productsBySource, recentProducts, sources as sourcesQuery, topMovers, } from '../lib/queries'; +import { ASIN_RE } from '../../../packages/schema/index.js'; import type { Product } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; @@ -44,26 +46,25 @@ import BoardTable from '../features/board/BoardTable'; import { boardColumns, dataColumnVisibility } from '../features/board/columns'; import '../features/board/board.css'; -const LIMITS = { movers: 100, source: 500, recent: 300 } as const; +const MOVERS_LIMIT = 100; const fmt = (n: number) => n.toLocaleString('en-US'); -/** Header count. Never presents a capped window as the whole workspace. */ -function countLabel( +/** Header count: how many there are, and how many of them are loaded. */ +export function countLabel( visible: number, loaded: number, - capped: boolean, total: number | null, + searching: boolean, ): string { - const noun = loaded === 1 ? 'product' : 'products'; - if (!capped || (total !== null && total <= loaded)) { - return visible === loaded ? `${fmt(loaded)} ${noun}` : `${fmt(visible)} of ${fmt(loaded)} ${noun}`; + const noun = (n: number) => (n === 1 ? 'product' : 'products'); + const all = total !== null && loaded >= total; + if (searching) { + const scope = all ? fmt(loaded) : `${fmt(loaded)} loaded`; + return `${fmt(visible)} ${visible === 1 ? 'match' : 'matches'} in ${scope} ${noun(loaded)}`; } - const span = - total !== null - ? `first ${fmt(loaded)} of ${fmt(total)} ${noun}` - : `first ${fmt(loaded)} ${noun}, more not loaded`; - return visible === loaded ? `Showing the ${span}` : `${fmt(visible)} matches in the ${span}`; + if (all || total === null) return `${fmt(total ?? loaded)} ${noun(total ?? loaded)}`; + return `${fmt(loaded)} of ${fmt(total)} ${noun(total)} loaded`; } export default function Products() { @@ -79,30 +80,22 @@ export default function Products() { const [search, setSearch] = useState(''); const [sorting, setSorting] = useState([]); - const limit = - view === 'movers' ? LIMITS.movers : sourceId ? LIMITS.source : LIMITS.recent; + const scopeQuery = () => { + if (!wid) return null; + return sourceId ? productsBySource(wid, sourceId, null) : recentProducts(wid, null); + }; - const products = useSnapshotQuery( - () => { - if (!wid) return null; - if (view === 'movers') return topMovers(wid, limit); - if (sourceId) return productsBySource(wid, sourceId, limit); - return recentProducts(wid, limit); - }, + const latest = usePagedQuery( + () => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId], - view === 'movers' ? 'board:movers' : sourceId ? 'board:by-source' : 'board:recent', + (p) => p.asin, ); + const total = useServerCount(() => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId]); - // Only ask the server for a total once the window is actually full. - const capped = products.data.length >= limit; - const total = useServerCount( - () => { - if (!wid || !capped) return null; - if (view === 'movers') return topMovers(wid, null); - if (sourceId) return productsBySource(wid, sourceId, null); - return recentProducts(wid, null); - }, - [wid, view, sourceId, capped], + const movers = useSnapshotQuery( + () => (wid && view === 'movers' ? topMovers(wid, MOVERS_LIMIT) : null), + [wid, view], + 'board:movers', ); const sourcesState = useSnapshotQuery( @@ -111,29 +104,41 @@ export default function Products() { 'board:sources', ); + const loaded = view === 'movers' ? movers.data : latest.data; + const state = view === 'movers' ? movers : latest; + // Movers shows only rows that actually carry a delta; a source scope in // movers view narrows client-side (the movers query is global). const scopedRows = useMemo(() => { - if (view !== 'movers') return products.data; - let list = products.data.filter( - (p) => p.delta?.pPct !== undefined || p.delta?.p !== undefined, - ); + if (view !== 'movers') return loaded; + let list = loaded.filter((p) => p.delta?.pPct !== undefined || p.delta?.p !== undefined); if (sourceId) list = list.filter((p) => p.sourceIds?.includes(sourceId)); return list; - }, [products.data, view, sourceId]); + }, [loaded, view, sourceId]); - // Search-in-loaded-set: name or ASIN, case-insensitive. + // An exact ASIN is read directly, so search reaches past the loaded pages. const query = search.trim().toLowerCase(); + const exactAsin = ASIN_RE.test(search.trim().toUpperCase()) ? search.trim().toUpperCase() : null; + const exactRef = useMemo( + () => (wid && exactAsin ? productRef(wid, exactAsin) : null), + [wid, exactAsin], + ); + const exact = useDocOnce(exactRef); + const rows = useMemo(() => { if (!query) return scopedRows; - return scopedRows.filter( + const hits = scopedRows.filter( (p) => p.asin.toLowerCase().includes(query) || (p.name ?? '').toLowerCase().includes(query), ); - }, [scopedRows, query]); + const found = exact.data; + const inScope = found && (!sourceId || found.sourceIds?.includes(sourceId)); + if (found && inScope && !hits.some((p) => p.asin === found.asin)) return [found, ...hits]; + return hits; + }, [scopedRows, query, exact.data, sourceId]); - const columnVisibility = useMemo(() => dataColumnVisibility(products.data), [products.data]); + const columnVisibility = useMemo(() => dataColumnVisibility(loaded), [loaded]); const table = useReactTable({ data: rows, @@ -177,15 +182,17 @@ export default function Products() { return s ? sourceLabel(s) : sourceId; }, [sourceId, sourcesState.data]); - const loading = !wid || products.loading; + const loading = !wid || state.loading; const subtitle = loading ? 'Loading the board…' : [ - countLabel(rows.length, scopedRows.length, capped && !(view === 'movers' && sourceId), total), + view === 'movers' + ? `${fmt(scopedRows.length)} movers` + : countLabel(rows.length, scopedRows.length, total, query !== ''), view === 'movers' ? sourceId - ? `biggest drops within the global top ${limit}` + ? `biggest drops within the global top ${MOVERS_LIMIT}` : 'biggest price drops first' : 'latest observations', sourceName ? `source: ${sourceName}` : null, @@ -202,9 +209,9 @@ export default function Products() {
); - } else if (products.error) { + } else if (state.error) { body = ( - + ); } else if (scopedRows.length === 0) { if (view === 'movers') { @@ -248,7 +255,11 @@ export default function Products() { body = ( setSearch('')}> Clear search @@ -260,6 +271,8 @@ export default function Products() { body = ; } + const showMore = view === 'latest' && !loading && !state.error && latest.hasMore; + return ( <> } /> - + {body} + {showMore ? ( +
+ + + {total !== null + ? `${fmt(latest.data.length)} of ${fmt(total)} loaded` + : `${fmt(latest.data.length)} loaded`} + +
+ ) : null} {asin && wid ? ( ) : null} From e65682f7ed752030493f83a5f4f37f8bc3fdfd3d Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:14:17 -0400 Subject: [PATCH 08/28] Page the run inbox instead of capping it at 30 --- dashboard/src/features/runs/groupRuns.ts | 2 +- dashboard/src/lib/queries.ts | 17 ++++++++--- dashboard/src/routes/Runs.tsx | 37 +++++++++++++++++++----- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/dashboard/src/features/runs/groupRuns.ts b/dashboard/src/features/runs/groupRuns.ts index de7c173..58a42d9 100644 --- a/dashboard/src/features/runs/groupRuns.ts +++ b/dashboard/src/features/runs/groupRuns.ts @@ -1,7 +1,7 @@ // Date-bucketing for the Run Inbox: Today / This week / Earlier. // Buckets key off the run's UTC dayKey (falling back to startedAt) so the // grouping matches the dayKey shown on each card. Input arrives newest- -// first from runsRecent(); bucketing preserves that order. +// first from runsNewestFirst(); bucketing preserves that order. import type { Run } from '../../lib/types'; import { dayKey, dayKeyDiff, toMillis } from '../../lib/format'; diff --git a/dashboard/src/lib/queries.ts b/dashboard/src/lib/queries.ts index 768e198..be49468 100644 --- a/dashboard/src/lib/queries.ts +++ b/dashboard/src/lib/queries.ts @@ -3,7 +3,7 @@ // auditable in one file (docs/ops/billing-runbook.md): // * products are ALWAYS scoped (per-source / top-N / recent-N) — never // the whole collection; -// * runs list is capped at 30; +// * runs are paged 30 at a time; // * sources is the only whole-collection read (tiny by design); // * history + offerSnapshots are one-shot fetch material. @@ -94,12 +94,21 @@ export function productRef(wid: string, asin: string): DocumentReference { +/** Run Inbox — newest first, no limit: page it with usePagedQuery. */ +export function runsNewestFirst(wid: string): Query { return query( collection(db, 'workspaces', wid, 'runs').withConverter(runConverter), orderBy('startedAt', 'desc'), - limit(Math.min(lim, 30)), + ); +} + +/** A source's newest runs. Index: runs(sourceId asc, startedAt desc). */ +export function runsOfSource(wid: string, sourceId: string, lim: number): Query { + return query( + collection(db, 'workspaces', wid, 'runs').withConverter(runConverter), + where('sourceId', '==', sourceId), + orderBy('startedAt', 'desc'), + limit(lim), ); } diff --git a/dashboard/src/routes/Runs.tsx b/dashboard/src/routes/Runs.tsx index 606188b..4430f95 100644 --- a/dashboard/src/routes/Runs.tsx +++ b/dashboard/src/routes/Runs.tsx @@ -1,13 +1,14 @@ -// Run Inbox (MVP feature 2) — the 30 most recent scrape-run headers as -// instrument cards, newest first, grouped Today / This week / Earlier. +// Run Inbox (MVP feature 2) — scrape-run headers as instrument cards, +// newest first, 30 at a time, grouped Today / This week / Earlier. // Source names are joined client-side from the (tiny, sanctioned) sources // listener. Clicking a card opens the product board scoped to that // run's source: /?source=. import { Fragment, useMemo } from 'react'; import { useNavigate } from 'react-router-dom'; -import { useSnapshotQuery, useWorkspace } from '../lib/hooks'; -import { runsRecent, sources } from '../lib/queries'; +import { useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; +import { usePagedQuery } from '../lib/paging'; +import { runsNewestFirst, sources } from '../lib/queries'; import type { Source } from '../lib/types'; import PageHeader from '../components/PageHeader'; import EmptyState from '../components/EmptyState'; @@ -18,16 +19,21 @@ import { CWS_URL } from '../auth/SignIn'; import RunCard, { RunCardSkeleton } from '../features/runs/RunCard'; import { groupRuns } from '../features/runs/groupRuns'; import '../features/runs/runs.css'; +import '../features/board/board.css'; + +const RUNS_PAGE = 30; export default function Runs() { const { wid } = useWorkspace(); const navigate = useNavigate(); - const runsState = useSnapshotQuery( - () => (wid ? runsRecent(wid, 30) : null), + const runsState = usePagedQuery( + () => (wid ? runsNewestFirst(wid) : null), [wid], - 'runs:recent30', + (r) => r.runId, + RUNS_PAGE, ); + const total = useServerCount(() => (wid ? runsNewestFirst(wid) : null), [wid]); const sourcesState = useSnapshotQuery( () => (wid ? sources(wid) : null), [wid], @@ -96,10 +102,25 @@ export default function Runs() { <> {body} + {!loading && !runsState.error && runsState.hasMore ? ( +
+ + + {runsState.data.length} + {total !== null ? ` of ${total}` : ''} loaded + +
+ ) : null} ); } From bfb523c2ba61a8af48bea8ab907cc1abff8a68af Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:14:31 -0400 Subject: [PATCH 09/28] Group runs by the local day they ran, as the extension files them --- dashboard/src/features/runs/groupRuns.ts | 16 ++++++++-------- dashboard/src/lib/format.ts | 8 +++++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dashboard/src/features/runs/groupRuns.ts b/dashboard/src/features/runs/groupRuns.ts index 58a42d9..13a99ef 100644 --- a/dashboard/src/features/runs/groupRuns.ts +++ b/dashboard/src/features/runs/groupRuns.ts @@ -1,10 +1,10 @@ // Date-bucketing for the Run Inbox: Today / This week / Earlier. -// Buckets key off the run's UTC dayKey (falling back to startedAt) so the -// grouping matches the dayKey shown on each card. Input arrives newest- +// Buckets key off the run's dayKey, the local date where it ran (falling +// back to startedAt), so the grouping matches the date on each card. Input arrives newest- // first from runsNewestFirst(); bucketing preserves that order. import type { Run } from '../../lib/types'; -import { dayKey, dayKeyDiff, toMillis } from '../../lib/format'; +import { dayKeyDiff, localDayKey, toMillis } from '../../lib/format'; export type RunGroupKey = 'today' | 'week' | 'earlier'; @@ -20,15 +20,15 @@ export interface RunGroup { runs: Run[]; } -/** The run's UTC dayKey, derived from startedAt when absent. */ +/** The run's dayKey, derived from startedAt when absent. */ function runDayKey(run: Run): string | null { if (run.dayKey) return run.dayKey; const ms = toMillis(run.startedAt); - return ms === null ? null : dayKey(new Date(ms)); + return ms === null ? null : localDayKey(new Date(ms)); } -/** today = same UTC day; week = 1–6 days ago; earlier = older / undated. */ -export function groupKeyFor(run: Run, todayKey: string = dayKey()): RunGroupKey { +/** today = same local day; week = 1–6 days ago; earlier = older / undated. */ +export function groupKeyFor(run: Run, todayKey: string = localDayKey()): RunGroupKey { const key = runDayKey(run); if (!key) return 'earlier'; const daysAgo = dayKeyDiff(key, todayKey); @@ -39,7 +39,7 @@ export function groupKeyFor(run: Run, todayKey: string = dayKey()): RunGroupKey } /** Bucket runs (already newest-first) into ordered, non-empty groups. */ -export function groupRuns(runs: Run[], todayKey: string = dayKey()): RunGroup[] { +export function groupRuns(runs: Run[], todayKey: string = localDayKey()): RunGroup[] { const buckets: Record = { today: [], week: [], earlier: [] }; for (const run of runs) buckets[groupKeyFor(run, todayKey)].push(run); return (['today', 'week', 'earlier'] as const) diff --git a/dashboard/src/lib/format.ts b/dashboard/src/lib/format.ts index 28c5be7..1b6df48 100644 --- a/dashboard/src/lib/format.ts +++ b/dashboard/src/lib/format.ts @@ -2,6 +2,7 @@ // and timestamps. All money flows through here as integer cents. import { Timestamp } from 'firebase/firestore'; +import { dayKeyOf } from '../../../packages/schema/index.js'; import type { Source } from './types'; const MS_PER_DAY = 86_400_000; @@ -68,11 +69,16 @@ export function relativeTime(at: TimeLike): string { return future ? `in ${label}` : `${label} ago`; } -/** UTC 'YYYY-MM-DD' for a date (default: now). Matches run/history dayKeys. */ +/** UTC 'YYYY-MM-DD' for a date (default: now). */ export function dayKey(date: Date = new Date()): string { return date.toISOString().slice(0, 10); } +/** Local 'YYYY-MM-DD', the way the extension writes run and history dayKeys. */ +export function localDayKey(date: Date = new Date()): string { + return dayKeyOf(date.getTime(), date.getTimezoneOffset()); +} + /** 'YYYY-MM-DD' -> Date at UTC midnight. */ export function dayKeyToDate(key: string): Date { return new Date(`${key}T00:00:00Z`); From 1b8ee0f5906b8d4350c18271315864daf2b895d5 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:15:36 -0400 Subject: [PATCH 10/28] Export the whole scope a page at a time, not just the loaded rows --- dashboard/src/features/board/BoardToolbar.tsx | 21 ++++++---- dashboard/src/features/board/sortRows.ts | 32 ++++++++++++++ .../src/features/export/ExportButton.tsx | 42 ++++++++++++------- dashboard/src/features/export/exportXlsx.ts | 6 +-- dashboard/src/lib/paging.ts | 22 ++++++++++ dashboard/src/routes/Products.tsx | 23 ++++++++-- 6 files changed, 115 insertions(+), 31 deletions(-) create mode 100644 dashboard/src/features/board/sortRows.ts diff --git a/dashboard/src/features/board/BoardToolbar.tsx b/dashboard/src/features/board/BoardToolbar.tsx index 68acb01..95e4442 100644 --- a/dashboard/src/features/board/BoardToolbar.tsx +++ b/dashboard/src/features/board/BoardToolbar.tsx @@ -1,7 +1,7 @@ -// Delta-board toolbar — search-in-loaded-set, Latest|Movers segmented -// toggle (gold active), source scope dropdown (synced to ?source=), the -// buyer-semantics legend popover and the XLSX export of the currently -// visible sorted rows. Pure presentation: all state lives in the route. +// Delta-board toolbar — search, Latest|Movers segmented toggle (gold +// active), source scope dropdown (synced to ?source=), the buyer-semantics +// legend popover and the XLSX export of the whole scope. Pure +// presentation: all state lives in the route. import { useEffect, useRef, useState } from 'react'; import { clsx } from 'clsx'; @@ -83,8 +83,10 @@ export interface BoardToolbarProps { sourceId: string | null; sources: Source[]; onSourceChange: (sourceId: string | null) => void; - /** Currently visible rows in sorted order — exactly what Export writes. */ - exportRows: Product[]; + /** Fetches every row Export writes: the whole scope, searched and sorted. */ + exportLoad: (onProgress: (loaded: number) => void) => Promise; + /** Rows Export will write, when known before fetching. */ + exportCount: number | null; } export default function BoardToolbar({ @@ -95,7 +97,8 @@ export default function BoardToolbar({ sourceId, sources, onSourceChange, - exportRows, + exportLoad, + exportCount, }: BoardToolbarProps) { const options = [...sources].sort((a, b) => sourceLabel(a).localeCompare(sourceLabel(b)), @@ -112,7 +115,7 @@ export default function BoardToolbar({ value={search} onChange={(e) => onSearchChange(e.target.value)} placeholder="Search name or ASIN" - aria-label="Search loaded products by name or ASIN" + aria-label="Search products by name or ASIN" spellCheck={false} />
@@ -150,7 +153,7 @@ export default function BoardToolbar({ ))} - +
); } diff --git a/dashboard/src/features/board/sortRows.ts b/dashboard/src/features/board/sortRows.ts new file mode 100644 index 0000000..c056721 --- /dev/null +++ b/dashboard/src/features/board/sortRows.ts @@ -0,0 +1,32 @@ +// Sorts rows the way the board's table does, for rows the table never +// held (the full export). Uses the same accessors, with missing values last. + +import type { SortingState } from '@tanstack/react-table'; +import type { Product } from '../../lib/types'; +import { boardColumns } from './columns'; + +type Accessor = (p: Product) => unknown; + +function accessorOf(id: string): Accessor | null { + const col = boardColumns.find((c) => c.id === id) as { accessorFn?: (p: Product, i: number) => unknown } | undefined; + return col?.accessorFn ? (p) => col.accessorFn!(p, 0) : null; +} + +export function sortRows(rows: Product[], sorting: SortingState): Product[] { + const keys = sorting + .map((s) => ({ get: accessorOf(s.id), desc: s.desc })) + .filter((k): k is { get: Accessor; desc: boolean } => k.get !== null); + if (keys.length === 0) return rows; + return [...rows].sort((a, b) => { + for (const { get, desc } of keys) { + const x = get(a); + const y = get(b); + if (x === y) continue; + if (x === undefined || x === null) return 1; + if (y === undefined || y === null) return -1; + const cmp = typeof x === 'string' && typeof y === 'string' ? x.localeCompare(y) : (x as number) < (y as number) ? -1 : 1; + return desc ? -cmp : cmp; + } + return 0; + }); +} diff --git a/dashboard/src/features/export/ExportButton.tsx b/dashboard/src/features/export/ExportButton.tsx index fb8949f..0c66ced 100644 --- a/dashboard/src/features/export/ExportButton.tsx +++ b/dashboard/src/features/export/ExportButton.tsx @@ -1,6 +1,7 @@ -// Gold "Export XLSX" action for table views. Exports the already-loaded -// rows via exportProductsXlsx (which dynamic-imports xlsx on first click). -// Props contract is FROZEN: { rows, disabled? }. +// Gold "Export XLSX" action for table views. `load` fetches the rows to +// write, a page at a time when the scope is bigger than what is loaded, +// then exportProductsXlsx writes them (it dynamic-imports xlsx on first +// click). `count` is what the badge shows, when it is known up front. import { useEffect, useRef, useState } from 'react'; import Button from '../../components/Button'; @@ -10,7 +11,8 @@ import { reportError } from '../../lib/errors'; import './export.css'; export interface ExportButtonProps { - rows: Product[]; + load: (onProgress: (loaded: number) => void) => Promise; + count: number | null; disabled?: boolean; } @@ -18,8 +20,10 @@ type Phase = 'idle' | 'working' | 'done'; const CONFIRM_MS = 1500; -export default function ExportButton({ rows, disabled }: ExportButtonProps) { +export default function ExportButton({ load, count, disabled }: ExportButtonProps) { const [phase, setPhase] = useState('idle'); + const [progress, setProgress] = useState(0); + const [written, setWritten] = useState(0); const timerRef = useRef(null); const mountedRef = useRef(true); @@ -32,16 +36,21 @@ export default function ExportButton({ rows, disabled }: ExportButtonProps) { }, []); const handleClick = async () => { - if (phase === 'working' || rows.length === 0) return; + if (phase === 'working' || count === 0) return; if (timerRef.current !== null) { window.clearTimeout(timerRef.current); timerRef.current = null; } setPhase('working'); + setProgress(0); try { - // Async-aware: the first click also awaits the xlsx chunk download. + const rows = await load((n) => { + if (mountedRef.current) setProgress(n); + }); + if (rows.length === 0) throw new Error('nothing to export'); await exportProductsXlsx(rows); if (!mountedRef.current) return; + setWritten(rows.length); setPhase('done'); timerRef.current = window.setTimeout(() => { timerRef.current = null; @@ -53,7 +62,8 @@ export default function ExportButton({ rows, disabled }: ExportButtonProps) { } }; - const empty = rows.length === 0; + const empty = count === 0; + const fmt = (n: number) => n.toLocaleString('en-US'); return ( diff --git a/dashboard/src/features/export/exportXlsx.ts b/dashboard/src/features/export/exportXlsx.ts index 271c407..12ff775 100644 --- a/dashboard/src/features/export/exportXlsx.ts +++ b/dashboard/src/features/export/exportXlsx.ts @@ -1,6 +1,6 @@ -// XLSX export — writes the ALREADY-LOADED view rows to an FBA-lead-list -// workbook (zero extra Firestore reads — read-cost hygiene). The heavy -// `xlsx` (SheetJS) package is loaded via dynamic import() inside +// XLSX export — writes the rows ExportButton fetched (the whole scope, read +// a page at a time) to an FBA-lead-list workbook. The heavy `xlsx` +// (SheetJS) package is loaded via dynamic import() inside // exportProductsXlsx so it stays out of the main chunk; the row-shaping // logic below (buildRows / defaultFilename) is pure and testable. diff --git a/dashboard/src/lib/paging.ts b/dashboard/src/lib/paging.ts index 348000b..20e485e 100644 --- a/dashboard/src/lib/paging.ts +++ b/dashboard/src/lib/paging.ts @@ -128,3 +128,25 @@ export function usePagedQuery( loadMore, }; } + +/** Every document of `q`, read `size` at a time with a cursor. For export. */ +export async function fetchAll( + q: Query, + onProgress?: (loaded: number) => void, + size = 500, +): Promise<{ rows: T[]; invalid: SchemaProblem[] }> { + const rows: T[] = []; + const invalid: SchemaProblem[] = []; + let cursor: QueryDocumentSnapshot | null = null; + for (;;) { + const page: Query = cursor ? query(q, startAfter(cursor), limit(size)) : query(q, limit(size)); + const snap = await getDocs(page); + const part: Page = toPage(snap.docs, size); + rows.push(...part.rows); + invalid.push(...part.invalid); + onProgress?.(rows.length); + if (!part.full || !part.last) break; + cursor = part.last; + } + return { rows, invalid }; +} diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index f4c6281..406a023 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -20,7 +20,8 @@ import { type SortingState, } from '@tanstack/react-table'; import { useDocOnce, useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; -import { usePagedQuery, PAGE_SIZE } from '../lib/paging'; +import { fetchAll, usePagedQuery, PAGE_SIZE } from '../lib/paging'; +import { reportError } from '../lib/errors'; import { productRef, productsBySource, @@ -44,6 +45,7 @@ import BoardToolbar, { } from '../features/board/BoardToolbar'; import BoardTable from '../features/board/BoardTable'; import { boardColumns, dataColumnVisibility } from '../features/board/columns'; +import { sortRows } from '../features/board/sortRows'; import '../features/board/board.css'; const MOVERS_LIMIT = 100; @@ -150,8 +152,20 @@ export default function Products() { getRowId: (p) => p.asin, }); - // What the user currently sees, in sorted order — the export contract. - const visibleSortedRows = table.getRowModel().rows.map((r) => r.original); + // Export writes the whole scope, not the loaded pages: every product, + // searched and sorted like the table. Movers are all loaded already. + const matches = (p: Product) => + !query || p.asin.toLowerCase().includes(query) || (p.name ?? '').toLowerCase().includes(query); + const exportLoad = async (onProgress: (n: number) => void): Promise => { + const base = scopeQuery(); + if (view === 'movers' || !base || !latest.hasMore) return table.getRowModel().rows.map((r) => r.original); + const { rows: all, invalid } = await fetchAll(base, onProgress); + if (invalid.length > 0) { + reportError('export some rows', new Error(`${invalid.length} failed the schema check and were left out`)); + } + return sortRows(all.filter(matches), sorting); + }; + const exportCount = view === 'movers' || !latest.hasMore ? rows.length : query ? null : total; /* ── URL writers ──────────────────────────────────────────────── */ @@ -287,7 +301,8 @@ export default function Products() { sourceId={sourceId} sources={sourcesState.data} onSourceChange={changeSource} - exportRows={visibleSortedRows} + exportLoad={exportLoad} + exportCount={exportCount} /> } /> From 9aeeecc85406e0b2df4bce909d4b8ebd5f726c39 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:16:41 -0400 Subject: [PATCH 11/28] Compare a source's latest two runs page by page --- dashboard/src/lib/compare.ts | 75 +++++++++++++++++++++++++++++++++ dashboard/test/compare.test.mjs | 53 +++++++++++++++++++++++ package.json | 3 +- 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 dashboard/src/lib/compare.ts create mode 100644 dashboard/test/compare.test.mjs diff --git a/dashboard/src/lib/compare.ts b/dashboard/src/lib/compare.ts new file mode 100644 index 0000000..0d5a61d --- /dev/null +++ b/dashboard/src/lib/compare.ts @@ -0,0 +1,75 @@ +// Deltas and movers for one source, from its latest two runs. The page +// chunks of each run hold every ASIN the run saw with its point, so two +// runs of the same source compare like with like: the product document's +// own delta is against whatever run saw it last, from any source. +// +// Pure: page documents in, comparisons out. + +import { deltaOf, timeMs, type Delta, type Point } from '../../../packages/schema/index.js'; +import type { PageDoc, Product, Run } from './types'; + +const DAY_MS = 86400000; + +export interface Comparison { + asin: string; + now: Point; + /** null when the earlier run did not see this ASIN */ + prev: Point | null; + delta: Delta | null; +} + +/** Every ASIN in a run's pages with its point, first page first. */ +export function itemsOf(pages: Pick[]): Map { + const items = new Map(); + for (const page of [...pages].sort((a, b) => a.page - b.page)) { + for (const [asin, pt] of Object.entries(page.items)) { + if (items.has(asin)) continue; + const { sp: _sp, ...point } = pt; + items.set(asin, point); + } + } + return items; +} + +/** Whole days from the earlier run's start to the later one's. */ +export function daysBetween(latest: Pick, prev: Pick): number | null { + const a = timeMs(latest.startedAt); + const b = timeMs(prev.startedAt); + return a === null || b === null ? null : Math.floor((a - b) / DAY_MS); +} + +/** Each ASIN of the latest run against the earlier run. */ +export function compareRuns( + latestPages: Pick[], + prevPages: Pick[] | null, + days: number | null = null, +): Map { + const now = itemsOf(latestPages); + const before = prevPages ? itemsOf(prevPages) : new Map(); + const out = new Map(); + for (const [asin, pt] of now) { + const prev = before.get(asin) ?? null; + out.set(asin, { asin, now: pt, prev, delta: prev ? deltaOf(pt, prev, days) : null }); + } + return out; +} + +/** ASINs whose price changed, biggest drop first. */ +export function moversOf(comparisons: Iterable): Comparison[] { + return [...comparisons] + .filter((c) => c.delta?.p !== undefined && c.delta.p !== 0) + .sort((a, b) => (a.delta!.pPct ?? 0) - (b.delta!.pPct ?? 0) || a.delta!.p! - b.delta!.p!); +} + +/** + * A board row for a product as this source's latest run saw it: its point + * and its delta against the source's previous run. + */ +export function asSeenBy(product: Product, cmp: Comparison, run: Pick): Product { + return { + ...product, + latest: { ...cmp.now, at: run.startedAt, runId: run.runId, dayKey: run.dayKey }, + prev: cmp.prev, + delta: cmp.delta, + }; +} diff --git a/dashboard/test/compare.test.mjs b/dashboard/test/compare.test.mjs new file mode 100644 index 0000000..49dfb82 --- /dev/null +++ b/dashboard/test/compare.test.mjs @@ -0,0 +1,53 @@ +// Unit tests for lib/compare.ts. Run: npm run test:unit +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { asSeenBy, compareRuns, daysBetween, itemsOf, moversOf } from '../src/lib/compare.ts'; + +const page = (n, items) => ({ page: n, items }); + +test('itemsOf keeps the first page an ASIN was on and drops the sponsored flag', () => { + const items = itemsOf([page(2, { B000000001: { p: 900, sp: 1 } }), page(1, { B000000001: { p: 1000, rk: 3, sp: 0 } })]); + assert.deepEqual(items.get('B000000001'), { p: 1000, rk: 3 }); +}); + +test('compareRuns diffs each ASIN of the latest run against the earlier one', () => { + const latest = [page(1, { B000000001: { p: 900, r: 4.5, v: 110 }, B000000002: { p: 500 } })]; + const prev = [page(1, { B000000001: { p: 1000, r: 4.5, v: 100 } })]; + const cmp = compareRuns(latest, prev, 7); + assert.deepEqual(cmp.get('B000000001').delta, { p: -100, pPct: -10, r: 0, v: 10, days: 7 }); + assert.equal(cmp.get('B000000002').prev, null, 'new to this source'); + assert.equal(cmp.get('B000000002').delta, null); +}); + +test('a price missing from either run gives no price delta, not a zero', () => { + const cmp = compareRuns([page(1, { B000000001: { v: 12 } })], [page(1, { B000000001: { p: 1000, v: 10 } })]); + assert.deepEqual(cmp.get('B000000001').delta, { v: 2 }); + assert.deepEqual(moversOf(cmp.values()), []); +}); + +test('with no earlier run nothing has a delta', () => { + const cmp = compareRuns([page(1, { B000000001: { p: 1 } })], null); + assert.equal(cmp.get('B000000001').delta, null); +}); + +test('moversOf lists price changes only, biggest drop first', () => { + const cmp = compareRuns( + [page(1, { B000000001: { p: 950 }, B000000002: { p: 500 }, B000000003: { p: 1100 }, B000000004: { p: 700 } })], + [page(1, { B000000001: { p: 1000 }, B000000002: { p: 1000 }, B000000003: { p: 1000 }, B000000004: { p: 700 } })], + ); + assert.deepEqual(moversOf(cmp.values()).map((c) => c.asin), ['B000000002', 'B000000001', 'B000000003']); +}); + +test('daysBetween counts whole days between run starts', () => { + const at = (ms) => ({ startedAt: { toMillis: () => ms } }); + assert.equal(daysBetween(at(8.5 * 86400000), at(1 * 86400000)), 7); +}); + +test('asSeenBy shows the run point and delta over the product document', () => { + const product = { asin: 'B000000001', mk: 'US', name: 'Mug', latest: { p: 1, runId: 'other' }, delta: { p: 5 } }; + const cmp = { asin: 'B000000001', now: { p: 900 }, prev: { p: 1000 }, delta: { p: -100, pPct: -10 } }; + const row = asSeenBy(product, cmp, { runId: 'k_mug_2', dayKey: '2026-06-09', startedAt: 't' }); + assert.equal(row.name, 'Mug'); + assert.deepEqual(row.latest, { p: 900, at: 't', runId: 'k_mug_2', dayKey: '2026-06-09' }); + assert.deepEqual(row.delta, { p: -100, pPct: -10 }); +}); diff --git a/package.json b/package.json index f2689e2..611a5d0 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", + "test:unit": "node --experimental-strip-types --no-warnings --test \"dashboard/test/*.test.mjs\"", "check:schema": "node scripts/check-schema.mjs", - "test": "npm run typecheck && npm run check:schema && npm run build && node scripts/check-bundle.mjs && npm run test:rules" + "test": "npm run typecheck && npm run check:schema && npm run test:unit && npm run build && node scripts/check-bundle.mjs && npm run test:rules" }, "dependencies": { "@fontsource-variable/archivo": "^5.2.8", From b8fc8306a426c0a2b9bb94e564c1f613a9db793f Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:18:17 -0400 Subject: [PATCH 12/28] Compute deltas and movers per source from its latest two runs Movers was the global top 100 by delta.pPct filtered on the client, so a source with real drops showed none (F-46). It now diffs each source's latest two runs page chunk by page chunk. A source-scoped board shows deltas against that source's previous run, on screen and in the export. --- dashboard/src/features/board/BoardToolbar.tsx | 2 +- dashboard/src/lib/movers.ts | 165 ++++++++++++++++++ dashboard/src/lib/queries.ts | 20 ++- dashboard/src/routes/Products.tsx | 94 ++++++---- 4 files changed, 243 insertions(+), 38 deletions(-) create mode 100644 dashboard/src/lib/movers.ts diff --git a/dashboard/src/features/board/BoardToolbar.tsx b/dashboard/src/features/board/BoardToolbar.tsx index 95e4442..530f467 100644 --- a/dashboard/src/features/board/BoardToolbar.tsx +++ b/dashboard/src/features/board/BoardToolbar.tsx @@ -131,7 +131,7 @@ export default function BoardToolbar({ type="button" className={clsx('board-seg__btn', view === 'movers' && 'board-seg__btn--on')} aria-pressed={view === 'movers'} - title="Biggest price drops across all sources" + title="Price changes between each source's latest two runs" onClick={() => onViewChange('movers')} > Movers diff --git a/dashboard/src/lib/movers.ts b/dashboard/src/lib/movers.ts new file mode 100644 index 0000000..43e7ba9 --- /dev/null +++ b/dashboard/src/lib/movers.ts @@ -0,0 +1,165 @@ +// Reads for run comparisons: a source's latest two run headers and their +// page chunks (one-shot, never listeners), and the product documents of +// the movers for names and images. Per source that is 2 + pages reads, +// plus one per mover shown. + +import { useEffect, useState } from 'react'; +import { getDocs } from 'firebase/firestore'; +import { splitChecked, type SchemaProblem } from './checked'; +import { asSeenBy, compareRuns, daysBetween, moversOf, type Comparison } from './compare'; +import { productsByAsin, runPages, runsOfSource } from './queries'; +import type { Product, Run, Source } from './types'; + +/** Sources compared when the board is not scoped to one. */ +export const MOVER_SOURCES = 8; +/** Movers joined to their product documents and shown. */ +export const MOVERS_SHOWN = 500; + +export interface SourceComparison { + sourceId: string; + latest: Run; + prev: Run | null; + byAsin: Map; +} + +async function compareSource( + wid: string, + sourceId: string, + invalid: SchemaProblem[], +): Promise { + const runsSnap = await getDocs(runsOfSource(wid, sourceId, 2)); + const runs = splitChecked(runsSnap.docs.map((d) => d.data())); + invalid.push(...runs.invalid); + const [latest, prev = null] = runs.valid; + if (!latest) return null; + const pagesOf = async (run: Run) => { + const snap = await getDocs(runPages(wid, run.runId)); + const pages = splitChecked(snap.docs.map((d) => d.data())); + invalid.push(...pages.invalid); + return pages.valid; + }; + const [latestPages, prevPages] = await Promise.all([pagesOf(latest), prev ? pagesOf(prev) : null]); + const days = prev ? daysBetween(latest, prev) : null; + return { sourceId, latest, prev, byAsin: compareRuns(latestPages, prevPages, days) }; +} + +interface State { + data: T; + invalid: SchemaProblem[]; + loading: boolean; + error: Error | null; +} + +/** The latest-two-runs comparison of each source in `sourceIds`. */ +export function useSourceComparisons( + wid: string | null, + sourceIds: string[] | null, +): State { + const key = sourceIds ? sourceIds.join('\n') : null; + const [state, setState] = useState>({ + data: [], + invalid: [], + loading: key !== null, + error: null, + }); + + useEffect(() => { + if (!wid || key === null) { + setState({ data: [], invalid: [], loading: false, error: null }); + return; + } + let cancelled = false; + setState({ data: [], invalid: [], loading: true, error: null }); + const invalid: SchemaProblem[] = []; + Promise.all(key.split('\n').filter(Boolean).map((id) => compareSource(wid, id, invalid))) + .then((all) => { + if (cancelled) return; + setState({ data: all.filter((c): c is SourceComparison => c !== null), invalid, loading: false, error: null }); + }) + .catch((error: Error) => { + console.error('[proscan] run comparison failed', error); + if (!cancelled) setState({ data: [], invalid, loading: false, error }); + }); + return () => { + cancelled = true; + }; + }, [wid, key]); + + return state; +} + +/** The sources Movers compares: the one in scope, else the most recently scanned. */ +export function moverSources(sourceId: string | null, sources: Source[]): string[] { + if (sourceId) return [sourceId]; + return [...sources] + .filter((s) => s.lastRunId) + .sort((a, b) => (b.lastScrapedAt?.toMillis() ?? 0) - (a.lastScrapedAt?.toMillis() ?? 0)) + .slice(0, MOVER_SOURCES) + .map((s) => s.sourceId); +} + +export interface MoverRows { + rows: Product[]; + /** price changes found, before the MOVERS_SHOWN cap */ + found: number; + compared: SourceComparison[]; +} + +/** + * Movers across the given comparisons, biggest drop first, as board rows: + * each ASIN as its source's latest run saw it. An ASIN in several sources + * shows under the most recently scanned one. + */ +export function useMoverRows(wid: string | null, comparisons: State): State { + const [state, setState] = useState>({ + data: { rows: [], found: 0, compared: [] }, + invalid: [], + loading: true, + error: null, + }); + + useEffect(() => { + if (!wid || comparisons.loading || comparisons.error) { + setState({ data: { rows: [], found: 0, compared: [] }, invalid: [], loading: comparisons.loading, error: comparisons.error }); + return; + } + const picked = new Map(); + for (const sc of comparisons.data) { + for (const cmp of moversOf(sc.byAsin.values())) { + if (!picked.has(cmp.asin)) picked.set(cmp.asin, { cmp, run: sc.latest }); + } + } + const ordered = moversOf([...picked.values()].map((p) => p.cmp)); + const shown = ordered.slice(0, MOVERS_SHOWN); + let cancelled = false; + setState((s) => ({ ...s, loading: true, error: null })); + + const chunks: string[][] = []; + for (let i = 0; i < shown.length; i += 30) chunks.push(shown.slice(i, i + 30).map((c) => c.asin)); + Promise.all(chunks.map((asins) => getDocs(productsByAsin(wid, asins)))) + .then((snaps) => { + if (cancelled) return; + const { valid, invalid } = splitChecked(snaps.flatMap((s) => s.docs.map((d) => d.data()))); + const docs = new Map(valid.map((p) => [p.asin, p])); + const rows = shown.map((cmp) => { + const product = docs.get(cmp.asin) ?? { asin: cmp.asin, mk: 'US' }; + return asSeenBy(product, cmp, picked.get(cmp.asin)!.run); + }); + setState({ + data: { rows, found: ordered.length, compared: comparisons.data }, + invalid: [...comparisons.invalid, ...invalid], + loading: false, + error: null, + }); + }) + .catch((error: Error) => { + console.error('[proscan] mover products failed', error); + if (!cancelled) setState({ data: { rows: [], found: 0, compared: [] }, invalid: [], loading: false, error }); + }); + return () => { + cancelled = true; + }; + }, [wid, comparisons]); + + return state; +} diff --git a/dashboard/src/lib/queries.ts b/dashboard/src/lib/queries.ts index be49468..7ddc129 100644 --- a/dashboard/src/lib/queries.ts +++ b/dashboard/src/lib/queries.ts @@ -10,6 +10,7 @@ import { collection, doc, + documentId, getDocs, limit, orderBy, @@ -25,6 +26,7 @@ import { import { db } from '../firebase'; import { historyConverter, + pageConverter, productConverter, runConverter, sourceConverter, @@ -33,6 +35,7 @@ import type { HistoryDoc, LeadStage, OfferSnapshot, + PageDoc, Product, ProductEvent, ProductLead, @@ -74,13 +77,6 @@ export function productsBySource( return lim === null ? q : query(q, limit(lim)); } -/** Global Movers / Flip Radar: biggest price DROPS first (delta.pPct asc — - * most-negative = best buying opportunity). Single-field index. */ -export function topMovers(wid: string, lim: number | null = 100): Query { - const q = query(productsCol(wid), orderBy('delta.pPct', 'asc')); - return lim === null ? q : query(q, limit(lim)); -} - /** Most recently observed products across all sources. */ export function recentProducts(wid: string, lim: number | null = 200): Query { const q = query(productsCol(wid), orderBy('latest.at', 'desc')); @@ -112,6 +108,16 @@ export function runsOfSource(wid: string, sourceId: string, lim: number): Query< ); } +/** A run's page chunks, one-shot material for run comparisons. */ +export function runPages(wid: string, runId: string): Query { + return collection(db, 'workspaces', wid, 'runs', runId, 'pages').withConverter(pageConverter); +} + +/** Up to 30 products by ASIN (the `in` limit). */ +export function productsByAsin(wid: string, asins: string[]): Query { + return query(productsCol(wid), where(documentId(), 'in', asins.slice(0, 30))); +} + /** Whole sources collection (the watchlist spine — tiny by design; * staleness sorting happens client-side via format.staleness). */ export function sources(wid: string): Query { diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index 406a023..f92ebc3 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -7,9 +7,11 @@ // page is live, "Load more" pages on with a cursor (F-45) // total -> getCountFromServer on the same query, so the header always // says how many there are, not how many are loaded -// movers -> topMovers(wid, 100) -// plus the sanctioned tiny sources listener for the scope dropdown. An -// exact ASIN in the search box is looked up directly, loaded or not. +// movers -> each source's latest two runs and their page chunks +// (lib/movers.ts): price changes run against run, per source +// plus the sanctioned tiny sources listener for the scope dropdown. With a +// source in scope, Latest also shows deltas against that source's previous +// run. An exact ASIN in the search box is looked up directly. import { useMemo, useState, type ReactNode } from 'react'; import { useSearchParams } from 'react-router-dom'; @@ -27,8 +29,9 @@ import { productsBySource, recentProducts, sources as sourcesQuery, - topMovers, } from '../lib/queries'; +import { asSeenBy } from '../lib/compare'; +import { MOVERS_SHOWN, moverSources, useMoverRows, useSourceComparisons } from '../lib/movers'; import { ASIN_RE } from '../../../packages/schema/index.js'; import type { Product } from '../lib/types'; import PageHeader from '../components/PageHeader'; @@ -48,8 +51,6 @@ import { boardColumns, dataColumnVisibility } from '../features/board/columns'; import { sortRows } from '../features/board/sortRows'; import '../features/board/board.css'; -const MOVERS_LIMIT = 100; - const fmt = (n: number) => n.toLocaleString('en-US'); /** Header count: how many there are, and how many of them are loaded. */ @@ -94,29 +95,46 @@ export default function Products() { ); const total = useServerCount(() => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId]); - const movers = useSnapshotQuery( - () => (wid && view === 'movers' ? topMovers(wid, MOVERS_LIMIT) : null), - [wid, view], - 'board:movers', - ); - const sourcesState = useSnapshotQuery( () => (wid ? sourcesQuery(wid) : null), [wid], 'board:sources', ); - const loaded = view === 'movers' ? movers.data : latest.data; - const state = view === 'movers' ? movers : latest; + // Which sources to compare run against run: the one in scope, or for + // Movers across all sources the most recently scanned ones. + const compareIds = useMemo(() => { + if (view === 'movers') { + if (!sourceId && sourcesState.loading) return null; + return moverSources(sourceId, sourcesState.data); + } + return sourceId ? [sourceId] : null; + }, [view, sourceId, sourcesState.loading, sourcesState.data]); + const comparisons = useSourceComparisons(wid, compareIds); + const movers = useMoverRows(view === 'movers' ? wid : null, comparisons); + + // In a source's scope, Latest shows each product as that source's last + // run saw it, with the delta against the run before. + const overlay = view === 'latest' && sourceId ? comparisons.data[0] : undefined; + const latestRows = useMemo(() => { + if (!overlay) return latest.data; + return latest.data.map((p) => { + const cmp = overlay.byAsin.get(p.asin); + return cmp ? asSeenBy(p, cmp, overlay.latest) : p; + }); + }, [latest.data, overlay]); - // Movers shows only rows that actually carry a delta; a source scope in - // movers view narrows client-side (the movers query is global). - const scopedRows = useMemo(() => { - if (view !== 'movers') return loaded; - let list = loaded.filter((p) => p.delta?.pPct !== undefined || p.delta?.p !== undefined); - if (sourceId) list = list.filter((p) => p.sourceIds?.includes(sourceId)); - return list; - }, [loaded, view, sourceId]); + const scopedRows = view === 'movers' ? movers.data.rows : latestRows; + const loaded = scopedRows; + const state = + view === 'movers' + ? movers + : { + ...latest, + loading: latest.loading || (sourceId !== null && comparisons.loading), + error: latest.error ?? comparisons.error, + invalid: [...latest.invalid, ...comparisons.invalid], + }; // An exact ASIN is read directly, so search reaches past the loaded pages. const query = search.trim().toLowerCase(); @@ -163,7 +181,13 @@ export default function Products() { if (invalid.length > 0) { reportError('export some rows', new Error(`${invalid.length} failed the schema check and were left out`)); } - return sortRows(all.filter(matches), sorting); + const seen = overlay + ? all.map((p) => { + const cmp = overlay.byAsin.get(p.asin); + return cmp ? asSeenBy(p, cmp, overlay.latest) : p; + }) + : all; + return sortRows(seen.filter(matches), sorting); }; const exportCount = view === 'movers' || !latest.hasMore ? rows.length : query ? null : total; @@ -198,17 +222,27 @@ export default function Products() { const loading = !wid || state.loading; + const moversLine = () => { + const { found, compared } = movers.data; + const n = found > MOVERS_SHOWN ? `top ${fmt(MOVERS_SHOWN)} of ${fmt(found)} movers` : `${fmt(found)} ${found === 1 ? 'mover' : 'movers'}`; + const only = compared.length === 1 ? compared[0] : null; + if (only) { + return only.prev + ? `${n} · run of ${only.latest.dayKey} against ${only.prev.dayKey}` + : `${n} · only one run so far`; + } + return `${n} across ${compared.length} ${compared.length === 1 ? 'source' : 'sources'}, each run against the one before`; + }; + const subtitle = loading ? 'Loading the board…' : [ + view === 'movers' ? moversLine() : countLabel(rows.length, scopedRows.length, total, query !== ''), view === 'movers' - ? `${fmt(scopedRows.length)} movers` - : countLabel(rows.length, scopedRows.length, total, query !== ''), - view === 'movers' - ? sourceId - ? `biggest drops within the global top ${MOVERS_LIMIT}` - : 'biggest price drops first' - : 'latest observations', + ? 'biggest price drops first' + : overlay?.prev + ? `deltas against the run of ${overlay.prev.dayKey}` + : 'latest observations', sourceName ? `source: ${sourceName}` : null, ] .filter(Boolean) From 93dfd37f665afe573a2502f66a47d446cf08f57e Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:18:26 -0400 Subject: [PATCH 13/28] Drop the composite indexes no query uses --- firestore.indexes.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/firestore.indexes.json b/firestore.indexes.json index 3574969..9c7f4b4 100644 --- a/firestore.indexes.json +++ b/firestore.indexes.json @@ -4,20 +4,8 @@ { "fieldPath": "sourceId", "order": "ASCENDING" }, { "fieldPath": "startedAt", "order": "DESCENDING" } ] }, - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "lead.stage", "order": "ASCENDING" }, - { "fieldPath": "lead.stageChangedAt", "order": "DESCENDING" } ] }, - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ { "fieldPath": "sourceIds", "arrayConfig": "CONTAINS" }, - { "fieldPath": "latest.at", "order": "DESCENDING" } ] }, - - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "sourceIds", "arrayConfig": "CONTAINS" }, - { "fieldPath": "delta.pPct", "order": "ASCENDING" } ] }, - - { "collectionGroup": "products", "queryScope": "COLLECTION", "fields": [ - { "fieldPath": "tags", "arrayConfig": "CONTAINS" }, { "fieldPath": "latest.at", "order": "DESCENDING" } ] } ], "fieldOverrides": [ From 3612e228ed7b0c6e6049ecd319d56b735f394141 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:18:43 -0400 Subject: [PATCH 14/28] Keep loading while the only answer is an empty cache --- dashboard/src/lib/hooks.ts | 3 +++ dashboard/src/lib/paging.ts | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dashboard/src/lib/hooks.ts b/dashboard/src/lib/hooks.ts index e2271a0..b73d344 100644 --- a/dashboard/src/lib/hooks.ts +++ b/dashboard/src/lib/hooks.ts @@ -125,7 +125,10 @@ export function useSnapshotQuery( let mountReads = 0; const unsubscribe = onSnapshot( q, + { includeMetadataChanges: true }, (snap) => { + // An empty answer from the cache is not "nothing there" (F-48). + if (snap.empty && snap.metadata.fromCache) return; if (firstSnapshot) { firstSnapshot = false; scanEnd(token); diff --git a/dashboard/src/lib/paging.ts b/dashboard/src/lib/paging.ts index 20e485e..98501c4 100644 --- a/dashboard/src/lib/paging.ts +++ b/dashboard/src/lib/paging.ts @@ -70,9 +70,15 @@ export function usePagedQuery( setLoadingMore(false); setActive(q !== null); if (!q) return; + // Metadata changes too, so the server's answer arrives even when it + // matches an empty cache (F-48). return onSnapshot( query(q, limit(size)), - (snap) => setFirst(toPage(snap.docs, size)), + { includeMetadataChanges: true }, + (snap) => { + if (snap.empty && snap.metadata.fromCache) return; + setFirst(toPage(snap.docs, size)); + }, (err) => { console.error('[proscan] first page failed', err); setError(err); From 35314de63f21eb294d0e8410cacf784f1cab993b Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:19:48 -0400 Subject: [PATCH 15/28] Let Google sign-ins set a password for the extension The extension only signs in with email and password, so a dashboard account made with Google could never sync (F-50). A card above every view links an email credential to the same account, confirming the Google session first when Firebase asks for a recent sign-in. --- dashboard/src/auth/ExtensionPassword.tsx | 217 +++++++++++++++++++++++ dashboard/src/auth/extpw.css | 81 +++++++++ dashboard/src/shell/Layout.tsx | 2 + 3 files changed, 300 insertions(+) create mode 100644 dashboard/src/auth/ExtensionPassword.tsx create mode 100644 dashboard/src/auth/extpw.css diff --git a/dashboard/src/auth/ExtensionPassword.tsx b/dashboard/src/auth/ExtensionPassword.tsx new file mode 100644 index 0000000..1c02ad9 --- /dev/null +++ b/dashboard/src/auth/ExtensionPassword.tsx @@ -0,0 +1,217 @@ +// "Set a password for the extension" (F-50). The extension signs in with +// email and password only, so an account made with Google has nothing to +// type there and its workspace never fills. Linking an email credential to +// the same account gives it a password without making a second account. + +import { useState, type FormEvent } from 'react'; +import { + EmailAuthProvider, + linkWithCredential, + reauthenticateWithPopup, + type User, +} from 'firebase/auth'; +import { FirebaseError } from 'firebase/app'; +import { googleProvider } from '../firebase'; +import { useAuthUser } from '../lib/hooks'; +import Button from '../components/Button'; +import './extpw.css'; + +export const MIN_PASSWORD = 8; + +export function hasPassword(user: User): boolean { + return user.providerData.some((p) => p.providerId === 'password'); +} + +function message(err: unknown): string { + const code = err instanceof FirebaseError ? err.code : ''; + switch (code) { + case 'auth/weak-password': + return `Use at least ${MIN_PASSWORD} characters.`; + case 'auth/credential-already-in-use': + case 'auth/email-already-in-use': + return 'Another ProScan account already uses this email with a password. Sign in to the extension with that password, or reset it from the extension.'; + case 'auth/popup-blocked': + return 'Your browser blocked the Google window. Allow pop-ups and try again.'; + case 'auth/popup-closed-by-user': + case 'auth/cancelled-popup-request': + return 'Google sign-in was closed before it finished. Try again.'; + case 'auth/network-request-failed': + return 'Network error. Check your connection and try again.'; + default: + return code ? `Couldn't set the password (${code}).` : "Couldn't set the password."; + } +} + +async function link(user: User, password: string): Promise { + const credential = EmailAuthProvider.credential(user.email!, password); + try { + await linkWithCredential(user, credential); + } catch (err) { + if (err instanceof FirebaseError && err.code === 'auth/provider-already-linked') return; + if (!(err instanceof FirebaseError) || err.code !== 'auth/requires-recent-login') throw err; + // An old Google session: confirm it, then link. + await reauthenticateWithPopup(user, googleProvider); + await linkWithCredential(user, EmailAuthProvider.credential(user.email!, password)); + } + await user.reload(); +} + +const DISMISS_KEY = (uid: string) => `proscan:extension-password-later:${uid}`; + +function readLater(uid: string): boolean { + try { + return window.localStorage.getItem(DISMISS_KEY(uid)) === '1'; + } catch { + return false; + } +} + +function writeLater(uid: string, later: boolean): void { + try { + if (later) window.localStorage.setItem(DISMISS_KEY(uid), '1'); + else window.localStorage.removeItem(DISMISS_KEY(uid)); + } catch { + /* storage blocked: the card just comes back next visit */ + } +} + +/** Shown above every view to a signed-in user who has no password yet. */ +export default function ExtensionPassword() { + const { user } = useAuthUser(); + const [open, setOpen] = useState(false); + const [later, setLater] = useState(() => (user ? readLater(user.uid) : false)); + const [password, setPassword] = useState(''); + const [confirm, setConfirm] = useState(''); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const [done, setDone] = useState(false); + + if (!user || !user.email) return null; + if (done) { + return ( +
+ Password set. In the extension, sign in with {user.email} and + this password. Your scans will sync here. + +
+ ); + } + if (hasPassword(user)) return null; + + if (later && !open) { + return ( +
+ The extension needs a password for {user.email}. + +
+ ); + } + + const submit = async (e: FormEvent) => { + e.preventDefault(); + if (busy) return; + setError(null); + if (password.length < MIN_PASSWORD) { + setError(`Use at least ${MIN_PASSWORD} characters.`); + return; + } + if (password !== confirm) { + setError("The two passwords don't match."); + return; + } + setBusy(true); + try { + await link(user, password); + setPassword(''); + setConfirm(''); + setOpen(false); + setDone(true); + } catch (err) { + console.error('[proscan] linking a password failed', err); + setError(message(err)); + } finally { + setBusy(false); + } + }; + + return ( +
+
+

+ Set a password for the extension +

+

+ The ProScan extension signs in with an email and password. You signed in here with + Google, so the extension has nothing to sign in with yet and no scans can reach this + dashboard. Set a password for {user.email}, then use it in + the extension. It stays the same account. +

+
+ {open ? ( +
+ + setPassword(e.target.value)} + placeholder={`At least ${MIN_PASSWORD} characters`} + /> + + setConfirm(e.target.value)} + /> + {error ? ( +
+ {error} +
+ ) : null} +
+ + +
+
+ ) : ( +
+ + +
+ )} +
+ ); +} diff --git a/dashboard/src/auth/extpw.css b/dashboard/src/auth/extpw.css new file mode 100644 index 0000000..3c996ad --- /dev/null +++ b/dashboard/src/auth/extpw.css @@ -0,0 +1,81 @@ +/* "Set a password for the extension" card, above the routed view. */ +.extpw { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: var(--space-4); + margin: 0 0 var(--space-4); + padding: var(--space-4); + background: var(--surface); + border: 1px solid var(--gold); + border-radius: var(--radius-md); + box-shadow: var(--shadow); + color: var(--ink); +} +.extpw--slim, +.extpw--done { + align-items: center; + gap: var(--space-2); + padding: var(--space-2) var(--space-3); + font-size: 13px; + border-color: var(--line); +} +.extpw--done { + border-color: var(--green); +} +.extpw__text { + flex: 1 1 320px; +} +.extpw__title { + margin: 0 0 var(--space-1); + font-size: 15px; + font-weight: 700; +} +.extpw__body { + margin: 0; + color: var(--muted); + font-size: 13px; + line-height: 1.5; +} +.extpw__form { + flex: 1 1 260px; + display: flex; + flex-direction: column; + gap: var(--space-1); + max-width: 320px; +} +.extpw__label { + font-size: 12px; + color: var(--muted); +} +.extpw__input { + height: 34px; + padding: 0 var(--space-2); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + font: inherit; + color: var(--ink); + background: var(--surface); +} +.extpw__input:focus { + outline: 2px solid var(--gold-glow); + border-color: var(--gold); +} +.extpw__error { + color: var(--red); + font-size: 13px; +} +.extpw__actions { + display: flex; + gap: var(--space-2); + margin-top: var(--space-2); +} +.extpw__link { + border: 0; + padding: 0; + background: none; + color: var(--ink); + font: inherit; + text-decoration: underline; + cursor: pointer; +} diff --git a/dashboard/src/shell/Layout.tsx b/dashboard/src/shell/Layout.tsx index 58992e1..de5d69a 100644 --- a/dashboard/src/shell/Layout.tsx +++ b/dashboard/src/shell/Layout.tsx @@ -2,6 +2,7 @@ import { Outlet, useLocation } from 'react-router-dom'; import Sidebar from './Sidebar'; import Topbar from './Topbar'; import Toaster from '../components/Toaster'; +import ExtensionPassword from '../auth/ExtensionPassword'; import './shell.css'; /** App shell grid: 232px chrome sidebar + topbar + light content area. @@ -16,6 +17,7 @@ export default function Layout() {
+
From 50be0ce961bdff734062869296b0820263ec547b Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:19:59 -0400 Subject: [PATCH 16/28] Ask for 8 characters when creating a password --- dashboard/src/auth/SignIn.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dashboard/src/auth/SignIn.tsx b/dashboard/src/auth/SignIn.tsx index d161943..d24592d 100644 --- a/dashboard/src/auth/SignIn.tsx +++ b/dashboard/src/auth/SignIn.tsx @@ -10,6 +10,7 @@ import { auth, googleProvider, USE_EMULATOR } from '../firebase'; import Button from '../components/Button'; import { RadarIcon } from '../components/EmptyState'; import GoogleButton from './GoogleButton'; +import { MIN_PASSWORD } from './ExtensionPassword'; import ScanBoard from './ScanBoard'; import './auth.css'; @@ -46,7 +47,7 @@ function friendlyAuthError(err: unknown): string { case 'auth/email-already-in-use': return 'An account with this email already exists — sign in instead.'; case 'auth/weak-password': - return 'Password must be at least 6 characters.'; + return `Password must be at least ${MIN_PASSWORD} characters.`; case 'auth/too-many-requests': return 'Too many attempts. Wait a minute and try again.'; case 'auth/network-request-failed': @@ -141,8 +142,12 @@ export default function SignIn() { const submit = async (e: FormEvent) => { e.preventDefault(); if (busy || googleBusy) return; - setBusy(true); clearMessages(); + if (mode === 'create' && password.length < MIN_PASSWORD) { + setError(`Password must be at least ${MIN_PASSWORD} characters.`); + return; + } + setBusy(true); try { if (mode === 'signin') { await signInWithEmailAndPassword(auth, email.trim(), password); @@ -260,7 +265,7 @@ export default function SignIn() { className="auth__input" type={showPw ? 'text' : 'password'} autoComplete={mode === 'signin' ? 'current-password' : 'new-password'} - placeholder={mode === 'create' ? 'At least 6 characters' : 'Enter your password'} + placeholder={mode === 'create' ? `At least ${MIN_PASSWORD} characters` : 'Enter your password'} value={password} onChange={(e) => setPassword(e.target.value)} required From 95a5fc27e34060b82d27e27491d13706a6e797ac Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:38:26 -0400 Subject: [PATCH 17/28] Set the password directly when linking says the account's own email is taken The Auth emulator checks every account for the email, including the one being linked, and refuses. With one account per email that holder can only be this account, so updatePassword adds the same email provider. --- dashboard/src/auth/ExtensionPassword.tsx | 29 ++++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/dashboard/src/auth/ExtensionPassword.tsx b/dashboard/src/auth/ExtensionPassword.tsx index 1c02ad9..44ea6a8 100644 --- a/dashboard/src/auth/ExtensionPassword.tsx +++ b/dashboard/src/auth/ExtensionPassword.tsx @@ -8,6 +8,7 @@ import { EmailAuthProvider, linkWithCredential, reauthenticateWithPopup, + updatePassword, type User, } from 'firebase/auth'; import { FirebaseError } from 'firebase/app'; @@ -23,7 +24,7 @@ export function hasPassword(user: User): boolean { } function message(err: unknown): string { - const code = err instanceof FirebaseError ? err.code : ''; + const code = codeOf(err); switch (code) { case 'auth/weak-password': return `Use at least ${MIN_PASSWORD} characters.`; @@ -42,16 +43,30 @@ function message(err: unknown): string { } } +const codeOf = (err: unknown) => (err instanceof FirebaseError ? err.code : ''); + +async function linkOnce(user: User, password: string): Promise { + try { + await linkWithCredential(user, EmailAuthProvider.credential(user.email!, password)); + } catch (err) { + if (codeOf(err) === 'auth/provider-already-linked') return; + // The account's own email counts as taken where the backend checks + // every account, as the Auth emulator does. With one account per email + // the only holder is this account, and a password set on it adds the + // same email provider. + if (codeOf(err) !== 'auth/email-already-in-use') throw err; + await updatePassword(user, password); + } +} + async function link(user: User, password: string): Promise { - const credential = EmailAuthProvider.credential(user.email!, password); try { - await linkWithCredential(user, credential); + await linkOnce(user, password); } catch (err) { - if (err instanceof FirebaseError && err.code === 'auth/provider-already-linked') return; - if (!(err instanceof FirebaseError) || err.code !== 'auth/requires-recent-login') throw err; - // An old Google session: confirm it, then link. + if (codeOf(err) !== 'auth/requires-recent-login') throw err; + // An old Google session: confirm it, then try again. await reauthenticateWithPopup(user, googleProvider); - await linkWithCredential(user, EmailAuthProvider.credential(user.email!, password)); + await linkOnce(user, password); } await user.reload(); } From f9545e76f95f0241ada77a100151cb3329be8f08 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:38:26 -0400 Subject: [PATCH 18/28] Add an e2e run: the extension's sync seeds the emulator, the production bundle is checked The real engine and sync.js write 1,050 products over three runs on different days. vite preview serves the emulator-flagged production bundle, and Playwright checks the promised columns, true counts, Load more, exact ASIN search, the full export, run-against-run movers, and a Google account setting a password the extension then signs in with. Amazon hosts are answered locally. --- .gitignore | 2 + package.json | 1 + scripts/e2e.mjs | 420 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+) create mode 100644 scripts/e2e.mjs diff --git a/.gitignore b/.gitignore index e3dddfb..941dc02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ firebase-debug.*.log* # Build output (generated by `npm run build`) dist/ +# Emulator bundle built by `npm run test:e2e` +dist-e2e/ # Firebase cache .firebase/ diff --git a/package.json b/package.json index 611a5d0..3ab956f 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "seed": "node scripts/seed-emulators.mjs", "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", + "test:e2e": "node scripts/emulators.mjs exec \"node scripts/e2e.mjs\"", "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"", "test:unit": "node --experimental-strip-types --no-warnings --test \"dashboard/test/*.test.mjs\"", "check:schema": "node scripts/check-schema.mjs", diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs new file mode 100644 index 0000000..1829941 --- /dev/null +++ b/scripts/e2e.mjs @@ -0,0 +1,420 @@ +// End to end: the extension's real engine and sync module (from its +// checkout) seed the emulator with 1,050 products over three runs on +// different days, then the production dashboard bundle, built against the +// emulators and served by `vite preview`, is signed in to and checked: +// every promised column fills, counts are true, Load more pages on, Movers +// compares runs, Export writes the whole scope, and a Google account can +// set a password the extension then signs in with (F-50). +// +// npm run test:e2e +// +// Runs inside `firebase emulators:exec` (scripts/emulators.mjs). No Amazon +// traffic: every amazon.com and media-amazon.com request is answered here. + +import { spawn, spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { chromium } from 'playwright'; +import * as XLSX from 'xlsx'; +import { initializeApp, deleteApp } from 'firebase/app'; +import { + getAuth, + connectAuthEmulator, + createUserWithEmailAndPassword, + GoogleAuthProvider, + signInWithCredential, + signInWithEmailAndPassword, +} from 'firebase/auth'; +import { getFirestore, connectFirestoreEmulator, terminate } from 'firebase/firestore'; +import { requireExtension, WEB_ROOT } from './lib/extension.mjs'; +import { asinOf, cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const AUTH_HOST = process.env.FIREBASE_AUTH_EMULATOR_HOST; +const FS_HOST = process.env.FIRESTORE_EMULATOR_HOST; +if (!AUTH_HOST || !FS_HOST) throw new Error('run through npm run test:e2e, which starts the emulators'); + +const PREVIEW_PORT = Number(process.env.E2E_PREVIEW_PORT || 4179); +const BASE = `http://localhost:${PREVIEW_PORT}/dashboard/`; +const OUT_DIR = resolve(WEB_ROOT, 'dist-e2e', 'dashboard'); +const SHOTS = resolve(WEB_ROOT, '.screenshots', 'e2e'); +const DEADLINE = Date.now() + 7 * 60 * 1000; +const DAY = 86400000; + +const EMAIL = `e2e-${Date.now()}@proscan.test`; +const PASSWORD = 'e2e-pass-1234'; +const STORE = 'https://www.amazon.com/s?me=A3K9XELT4QZ6M2&marketplaceID=ATVPDKIKX0DER'; +const KEYWORD = 'https://www.amazon.com/s?k=stainless+tumbler'; + +let failed = 0; +const check = (name, ok, detail = '') => { + if (!ok) failed++; + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` (${detail})` : ''}`); +}; +// Bugs that live in the extension: reported, not fixed here. Strict, so +// the mark has to come off once the extension fixes them. +let known = 0; +const knownFailure = (name, ok, detail = '') => { + if (ok) { + failed++; + console.log(` FAIL ${name} now passes; remove its known-failure mark`); + } else { + known++; + console.log(` KNOWN FAILURE ${name}${detail ? ` (${detail})` : ''}`); + } +}; +const fmt = (n) => n.toLocaleString('en-US'); + +function firebase(name) { + const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }, name); + const auth = getAuth(app); + const db = getFirestore(app); + connectAuthEmulator(auth, `http://${AUTH_HOST}`, { disableWarnings: true }); + const [h, p] = FS_HOST.split(':'); + connectFirestoreEmulator(db, h, Number(p)); + return { app, auth, db }; +} + +/* ── the scan data ───────────────────────────────────────────────── */ + +// Run 2 moves a third of the storefront down, a third up, keeps the rest, +// and loses every 10th price to a parse failure. +const run2Price = (i) => (i % 10 === 0 ? null : i % 3 === 0 ? 1000 + i - 50 * ((i % 7) + 1) : i % 3 === 1 ? 1000 + i + 25 : 1000 + i); +const STORE_N = 700; +const KEYWORD_FROM = 650; +const KEYWORD_N = 400; +const UNIQUE = KEYWORD_FROM + KEYWORD_N; // 1,050 +let expectedMovers = 0; +for (let i = 0; i < STORE_N; i++) { + const p = run2Price(i); + if (p !== null && p !== 1000 + i) expectedMovers++; +} + +async function seed() { + const ext = await loadExtension(requireExtension('the e2e seed')); + const fb = firebase('e2e-seed'); + const uid = (await createUserWithEmailAndPassword(fb.auth, EMAIL, PASSWORD)).user.uid; + const install = installExtension(ext, { uid, db: fb.db }); + const day1 = Date.parse('2026-09-01T14:00:00Z'); + const runs = [ + await install.scrape(STORE, day1, cards(STORE_N), 48), + await install.scrape(STORE, day1 + 7 * DAY, cards(STORE_N, { price: run2Price }), 48), + await install.scrape(KEYWORD, day1 + 8 * DAY, cards(KEYWORD_N, { offset: KEYWORD_FROM }), 48), + ]; + const t0 = Date.now(); + const totals = await install.flush(); + console.log(`[seed] ${JSON.stringify(totals)} in ${Math.round((Date.now() - t0) / 1000)}s`); + check('the extension synced three runs', totals.runs === 3 && (await install.pending()) === 0, JSON.stringify(totals)); + await terminate(fb.db); + await deleteApp(fb.app); + return { uid, runs, day1, ext }; +} + +/* ── the production bundle ───────────────────────────────────────── */ + +function build() { + rmSync(resolve(WEB_ROOT, 'dist-e2e'), { recursive: true, force: true }); + const res = spawnSync('npx', ['vite', 'build', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--emptyOutDir'], { + cwd: WEB_ROOT, + shell: true, + stdio: 'inherit', + env: { ...process.env, VITE_USE_EMULATOR: 'true' }, + timeout: 3 * 60 * 1000, + }); + if (res.status !== 0) throw new Error('dashboard build failed'); +} + +async function preview() { + const child = spawn( + 'npx', + ['vite', 'preview', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--port', String(PREVIEW_PORT), '--strictPort'], + { cwd: WEB_ROOT, shell: true, stdio: 'ignore' }, + ); + for (let i = 0; i < 60; i++) { + try { + const res = await fetch(BASE); + if (res.ok) return child; + } catch { + /* not up yet */ + } + await new Promise((r) => setTimeout(r, 500)); + } + kill(child); + throw new Error(`vite preview did not answer on ${BASE}`); +} + +function kill(child) { + if (!child || child.exitCode !== null) return; + if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(child.pid), '/T', '/F'], { stdio: 'ignore' }); + else child.kill('SIGTERM'); +} + +/* ── the browser ─────────────────────────────────────────────────── */ + +const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64'); + +async function newPage(browser, consoleErrors) { + const ctx = await browser.newContext({ viewport: { width: 1440, height: 1400 }, acceptDownloads: true }); + await ctx.route(/^https?:\/\/([^/]*\.)?(amazon\.com|media-amazon\.com)\//, (route) => + route.fulfill({ status: 200, contentType: 'image/png', body: PNG }), + ); + const page = await ctx.newPage(); + page.on('console', (msg) => { + if (msg.type() === 'error') consoleErrors.push(msg.text()); + }); + page.on('pageerror', (err) => consoleErrors.push(String(err))); + return { ctx, page }; +} + +async function signIn(page, email, password) { + await page.goto(BASE); + await page.waitForSelector('#auth-email', { timeout: 30000 }); + await page.fill('#auth-email', email); + await page.fill('#auth-password', password); + await page.click('.auth__cta'); +} + +const subtitle = (page) => page.locator('.page-header__sub').innerText(); + +async function waitSubtitle(page, re, timeout = 30000) { + await page.waitForFunction( + (src) => new RegExp(src).test(document.querySelector('.page-header__sub')?.textContent ?? ''), + re.source, + { timeout }, + ); + return subtitle(page); +} + +/** The rendered board as {headers, rows: [{Header: text}], imgs}. */ +function readBoard(page) { + return page.evaluate(() => { + const headers = [...document.querySelectorAll('.board-table thead th')].map((th) => th.textContent.trim()); + const rows = [...document.querySelectorAll('.board-row')].map((tr) => { + const cells = [...tr.querySelectorAll('td')]; + const out = {}; + headers.forEach((h, i) => (out[h] = (cells[i]?.textContent ?? '').trim())); + out.__img = tr.querySelector('img.board-prod__img')?.getAttribute('src') ?? ''; + return out; + }); + return { headers, rows }; + }); +} + +const header = (headers, prefix) => headers.find((h) => h.startsWith(prefix)); +const share = (rows, fn) => (rows.length === 0 ? 0 : rows.filter(fn).length / rows.length); + +async function checkColumns(page, label, { deltas }) { + const { headers, rows } = await readBoard(page); + const need = ['Product', 'Price', 'Δ Price', 'Rating', 'Reviews', 'Rank']; + const missing = need.filter((h) => !header(headers, h)); + check(`${label}: promised columns are shown`, missing.length === 0, missing.length ? `missing ${missing.join(', ')}` : headers.join(' | ')); + check(`${label}: rows render`, rows.length >= 20, `${rows.length} rows`); + const col = (h) => header(headers, h); + check(`${label}: every row has a name`, share(rows, (r) => /Test product \d+/.test(r[col('Product')])) === 1); + check(`${label}: every row has an image`, share(rows, (r) => r.__img.startsWith('https://m.media-amazon.com/')) === 1); + const price = share(rows, (r) => /^\$\d/.test(r[col('Price')])); + check(`${label}: price fills`, price >= 0.85, `${Math.round(price * 100)}%`); + check(`${label}: rating fills`, share(rows, (r) => /^\d\.\d/.test(r[col('Rating')])) === 1); + check(`${label}: reviews fill`, share(rows, (r) => /^\d/.test(r[col('Reviews')])) === 1); + check(`${label}: rank fills`, share(rows, (r) => /^#\d+/.test(r[col('Rank')])) === 1); + if (deltas) { + const d = share(rows, (r) => /\$\d/.test(r[col('Δ Price')])); + check(`${label}: Δ price fills where the price moved`, d >= deltas, `${Math.round(d * 100)}%`); + } + return { headers, rows }; +} + +async function main() { + mkdirSync(SHOTS, { recursive: true }); + console.log('[seed] the extension scrapes and syncs'); + const { ext } = await seed(); + + console.log('[build] production bundle against the emulators'); + build(); + const server = await preview(); + const browser = await chromium.launch({ executablePath: process.env.QA_CHROMIUM || undefined }); + const consoleErrors = []; + try { + const { ctx, page } = await newPage(browser, consoleErrors); + + console.log('[board] all sources, latest'); + await signIn(page, EMAIL, PASSWORD); + await page.waitForSelector('.board-row', { timeout: 40000 }); + let sub = await waitSubtitle(page, /of 1,050 products loaded/); + check('header gives the true total', sub.includes(`200 of ${fmt(UNIQUE)} products loaded`), sub); + await checkColumns(page, 'all sources', { deltas: 0 }); + check('no password card for an email account', (await page.locator('.extpw').count()) === 0); + await page.screenshot({ path: resolve(SHOTS, 'board.png') }); + + await page.click('.board-more button'); + sub = await waitSubtitle(page, /400 of 1,050/); + check('Load more pages on with a cursor', sub.includes(`400 of ${fmt(UNIQUE)}`), sub); + + const lookFor = asinOf(3); // storefront only, not on the loaded pages + await page.fill('.board-search', lookFor); + await page.waitForSelector(`.board-row:has-text("${lookFor}")`, { timeout: 15000 }); + check('an exact ASIN is found past the loaded pages', (await page.locator('.board-row').count()) === 1); + await page.fill('.board-search', ''); + + console.log('[export] whole scope'); + const [download] = await Promise.all([ + page.waitForEvent('download', { timeout: 60000 }), + page.click('button:has-text("Export XLSX")'), + ]); + const file = resolve(SHOTS, download.suggestedFilename()); + await download.saveAs(file); + const sheet = XLSX.read(readFileSync(file)).Sheets['ProScan Leads']; + const table = XLSX.utils.sheet_to_json(sheet, { defval: null }); + check('export writes every product, not the loaded pages', table.length === UNIQUE, `${table.length} rows`); + const filled = (key) => share(table, (r) => r[key] !== null && r[key] !== ''); + check('export: names', filled('Product Name') === 1); + check('export: prices', filled('Current Price') >= 0.85, `${Math.round(filled('Current Price') * 100)}%`); + check('export: ranks', filled('Rank') === 1); + check('export: source ids', filled('Source IDs') === 1); + check('export: last scanned', filled('Last Scanned') === 1); + + console.log('[board] one source: deltas against its previous run'); + await page.selectOption('.board-select', 's_A3K9XELT4QZ6M2'); + sub = await waitSubtitle(page, /deltas against the run of/); + // The 50 ASINs the keyword run also saw lose the storefront from sourceIds. + knownFailure('NEW-SYNC-1 (F-20 area): storefront scope counts all its products', sub.includes(`200 of ${fmt(STORE_N)} products loaded`), sub); + await checkColumns(page, 'storefront', { deltas: 0.5 }); + await page.screenshot({ path: resolve(SHOTS, 'board-storefront.png') }); + + console.log('[movers] run against run'); + await page.click('.board-seg__btn:has-text("Movers")'); + sub = await waitSubtitle(page, /movers · run of/); + check('storefront movers are counted run against run', sub.includes(`${fmt(expectedMovers)} movers`), `${sub} (want ${expectedMovers})`); + const movers = await checkColumns(page, 'movers', { deltas: 1 }); + const firstDelta = movers.rows[0]?.[header(movers.headers, 'Δ Price')] ?? ''; + check('the biggest drop is on top', firstDelta.startsWith('▼'), firstDelta); + await page.screenshot({ path: resolve(SHOTS, 'movers.png') }); + + await page.selectOption('.board-select', ''); + sub = await waitSubtitle(page, /across 2 sources/); + check('movers across all sources', sub.includes(`${fmt(expectedMovers)} movers across 2 sources`), sub); + + console.log('[drawer]'); + await page.goto(`${BASE}?asin=${asinOf(9)}`); + await page.waitForSelector('.drawer', { timeout: 20000 }); + await page.waitForSelector('.recharts-surface', { timeout: 20000 }); + check('drawer draws the price history', (await page.locator('.drawer [role="alert"]').count()) === 0); + await page.keyboard.press('Escape'); + + console.log('[runs] and [watchlist]'); + await page.click('.nav-link:has-text("Runs")'); + await page.waitForSelector('.run-card:not(.run-card--skeleton)', { timeout: 20000 }); + sub = await waitSubtitle(page, /3 runs/); + check('three run cards', (await page.locator('.run-card:not(.run-card--skeleton)').count()) === 3, sub); + await page.click('.nav-link:has-text("Watchlist")'); + await page.waitForSelector('.wl-row', { timeout: 20000 }); + check('two sources on the watchlist', (await page.locator('.wl-row').count()) === 2); + await ctx.close(); + + console.log('[F-50] a Google account sets a password for the extension'); + const googleErrors = []; + await googlePassword(browser, googleErrors, ext); + // The emulator refuses to link the account's own email with a 400 and + // the dashboard falls back to updatePassword; nothing else may log. + consoleErrors.push(...googleErrors.filter((t) => !/status of 400/.test(t))); + } finally { + await browser.close().catch(() => {}); + kill(server); + } + + const noise = consoleErrors.filter((t) => !/favicon/i.test(t)); + check('no console errors', noise.length === 0, noise.slice(0, 3).join(' | ')); +} + +async function googlePassword(browser, consoleErrors, ext) { + const email = `google-${Date.now()}@proscan.test`; + const { ctx, page } = await newPage(browser, consoleErrors); + // A Google sign-in, made in Node against the Auth emulator and handed to + // the page as its saved session. The emulator's popup relays through + // Google's gapi script from apis.google.com, which the test must not need. + const idp = firebase('e2e-google-idp'); + const token = JSON.stringify({ sub: `g${Date.now()}`, email, email_verified: true, name: 'Google Tester' }); + const google = await signInWithCredential(idp.auth, GoogleAuthProvider.credential(token)); + check('the account signs in with Google only', google.user.providerData.map((p) => p.providerId).join() === 'google.com'); + const saved = { key: 'firebase:authUser:demo-key:[DEFAULT]', value: google.user.toJSON() }; + await terminate(idp.db); + await deleteApp(idp.app); + + await page.goto(BASE); + await page.waitForSelector('.auth__cta', { timeout: 30000 }); + await page.evaluate( + ({ key, value }) => + new Promise((done, fail) => { + const req = indexedDB.open('firebaseLocalStorageDb'); + req.onerror = () => fail(req.error); + req.onsuccess = () => { + const tx = req.result.transaction('firebaseLocalStorage', 'readwrite'); + tx.objectStore('firebaseLocalStorage').put({ fbase_key: key, value }); + tx.oncomplete = () => done(); + tx.onerror = () => fail(tx.error); + }; + }), + saved, + ); + await page.reload(); + const offered = await page.waitForSelector('.extpw', { timeout: 30000 }).then(() => true, () => false); + if (!offered) { + await page.screenshot({ path: resolve(SHOTS, 'google-signin-failed.png') }); + const text = (await page.locator('body').innerText()).slice(0, 300).replace(/\s+/g, ' '); + check('a Google account is offered a password for the extension', false, text); + await ctx.close(); + return; + } + check('a Google account is offered a password for the extension', (await page.locator('#extpw-title').count()) === 1); + await page.screenshot({ path: resolve(SHOTS, 'extension-password.png') }); + + await page.click('.extpw button:has-text("Set a password")'); + await page.fill('#extpw-new', 'short'); + await page.fill('#extpw-confirm', 'short'); + await page.click('.extpw button:has-text("Save password")'); + check('a short password is refused', (await page.locator('.extpw__error').innerText()).includes('8 characters')); + const password = 'google-then-pass-1'; + await page.fill('#extpw-new', password); + await page.fill('#extpw-confirm', password); + await page.click('.extpw button:has-text("Save password")'); + const linked = await page.waitForSelector('.extpw--done', { timeout: 20000 }).then(() => true, () => false); + check('the password is linked', linked, linked ? '' : await page.locator('.extpw').innerText().catch(() => 'no card')); + if (!linked) { + await page.screenshot({ path: resolve(SHOTS, 'extension-password-failed.png') }); + await ctx.close(); + return; + } + + // The extension signs in with email and password: it now can, as the same account. + const fb = firebase('e2e-google'); + const cred = await signInWithEmailAndPassword(fb.auth, email, password); + const googleUid = await page.evaluate(() => document.querySelector('.sidebar__email')?.textContent ?? ''); + check('the extension can sign in with it', !!cred.user.uid && googleUid.includes(email), googleUid); + const install = installExtension(ext, { uid: cred.user.uid, db: fb.db }); + await install.scrape(KEYWORD, Date.parse('2026-09-12T10:00:00Z'), cards(5, { offset: 5000 }), 48); + await install.flush(); + await terminate(fb.db); + await deleteApp(fb.app); + + await page.reload(); + await page.waitForSelector('.board-row', { timeout: 30000 }); + const sub = await waitSubtitle(page, /products/); + check("and its scans reach the Google account's dashboard", sub.startsWith('5 products'), sub); + await ctx.close(); +} + +const timer = setTimeout(() => { + console.error('[e2e] over the time limit'); + process.exit(1); +}, DEADLINE - Date.now()); + +try { + await main(); +} catch (e) { + console.error('[fatal]', e); + failed++; +} finally { + clearTimeout(timer); + if (known) console.log(`\n${known} known failure(s), see above`); + console.log(failed === 0 ? '\nRESULT: PASS' : `\nRESULT: FAIL (${failed})`); + process.exit(failed === 0 ? 0 : 1); +} From 8df50a4d689ba49e3dded58e8bb3e7815c9614d1 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:38:49 -0400 Subject: [PATCH 19/28] Run the schema check, unit tests and e2e in CI against the extension's main branch --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++++- scripts/e2e.mjs | 11 +++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 974f3ac..ca0098a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,21 @@ permissions: jobs: check: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 25 + env: + PROSCAN_EXT: ${{ github.workspace }}/extension steps: - uses: actions/checkout@v4 + # The schema check, the sync rules test and the e2e run the + # extension's own schema, engine and sync module. Its main branch has + # to carry packages/schema (extension 2.3). + - uses: actions/checkout@v4 + with: + repository: EnesYilmazcode/AmazonSellerScraper + ref: main + path: extension + - uses: actions/setup-node@v4 with: node-version: 22 @@ -33,11 +44,19 @@ jobs: key: firebase-emulators-15.8.0 - run: npm ci + - run: npm ci + working-directory: extension - run: npm install -g firebase-tools@15.8.0 - name: Typecheck run: npm run typecheck + - name: Schema matches the extension + run: npm run check:schema + + - name: Unit tests + run: npm run test:unit + - name: Build run: npm run build @@ -54,3 +73,9 @@ jobs: env: VITE_USE_EMULATOR: 'true' run: npm run build && node scripts/check-bundle.mjs --emulator + + - name: Install Chromium for the e2e + run: npx playwright install --with-deps chromium + + - name: End to end (emulator, production bundle) + run: npm run test:e2e diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs index 1829941..d75aeda 100644 --- a/scripts/e2e.mjs +++ b/scripts/e2e.mjs @@ -127,7 +127,8 @@ async function preview() { const child = spawn( 'npx', ['vite', 'preview', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--port', String(PREVIEW_PORT), '--strictPort'], - { cwd: WEB_ROOT, shell: true, stdio: 'ignore' }, + // Its own process group off Windows, so kill() takes vite down with npx. + { cwd: WEB_ROOT, shell: true, stdio: 'ignore', detached: process.platform !== 'win32' }, ); for (let i = 0; i < 60; i++) { try { @@ -145,7 +146,13 @@ async function preview() { function kill(child) { if (!child || child.exitCode !== null) return; if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(child.pid), '/T', '/F'], { stdio: 'ignore' }); - else child.kill('SIGTERM'); + else { + try { + process.kill(-child.pid, 'SIGTERM'); + } catch { + child.kill('SIGTERM'); + } + } } /* ── the browser ─────────────────────────────────────────────────── */ From 4202d6f1f32dc944e2bb80f6ca6d43fab1e1dac8 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:39:29 -0400 Subject: [PATCH 20/28] Add a dev seed that writes through the extension's sync module --- package.json | 1 + scripts/seed-extension.mjs | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 scripts/seed-extension.mjs diff --git a/package.json b/package.json index 3ab956f..6ed1cd1 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "emulators": "node scripts/emulators.mjs start", "emulators:persist": "node scripts/emulators.mjs start --import=./.emulator-data --export-on-exit=./.emulator-data", "seed": "node scripts/seed-emulators.mjs", + "seed:extension": "node scripts/seed-extension.mjs", "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", "test:e2e": "node scripts/emulators.mjs exec \"node scripts/e2e.mjs\"", diff --git a/scripts/seed-extension.mjs b/scripts/seed-extension.mjs new file mode 100644 index 0000000..1cfeaa2 --- /dev/null +++ b/scripts/seed-extension.mjs @@ -0,0 +1,48 @@ +// Dev seed through the extension: its real engine and sync.js scan a +// storefront twice a week apart and a keyword once, into the running +// emulators, as dev@proscan.test / proscan-dev. Unlike seed-emulators.mjs +// (firebase-admin fixtures with Phase 5 fields), this is exactly what the +// extension writes. +// +// npm run emulators (one terminal) +// npm run seed:extension (another) + +process.env.FIRESTORE_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_FIRESTORE_PORT || 8080}`; +process.env.FIREBASE_AUTH_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_AUTH_PORT || 9099}`; + +import { initializeApp, deleteApp } from 'firebase/app'; +import { + getAuth, + connectAuthEmulator, + createUserWithEmailAndPassword, + signInWithEmailAndPassword, +} from 'firebase/auth'; +import { getFirestore, connectFirestoreEmulator, terminate } from 'firebase/firestore'; +import { requireExtension } from './lib/extension.mjs'; +import { cards, installExtension, loadExtension } from './lib/extension-sync.mjs'; + +const EMAIL = 'dev@proscan.test'; +const PASSWORD = 'proscan-dev'; +const DAY = 86400000; + +const ext = await loadExtension(requireExtension('the extension seed')); +const app = initializeApp({ projectId: 'demo-proscan', apiKey: 'demo-key' }); +const auth = getAuth(app); +const db = getFirestore(app); +connectAuthEmulator(auth, `http://${process.env.FIREBASE_AUTH_EMULATOR_HOST}`, { disableWarnings: true }); +const [host, port] = process.env.FIRESTORE_EMULATOR_HOST.split(':'); +connectFirestoreEmulator(db, host, Number(port)); + +const cred = await signInWithEmailAndPassword(auth, EMAIL, PASSWORD).catch(() => + createUserWithEmailAndPassword(auth, EMAIL, PASSWORD), +); +const install = installExtension(ext, { uid: cred.user.uid, db }); +const start = Date.now() - 8 * DAY; +const moved = (i) => (i % 10 === 0 ? null : i % 3 === 0 ? 1000 + i - 50 * ((i % 7) + 1) : 1000 + i); +await install.scrape('https://www.amazon.com/s?me=A3K9XELT4QZ6M2', start, cards(300), 48); +await install.scrape('https://www.amazon.com/s?me=A3K9XELT4QZ6M2', start + 7 * DAY, cards(300, { price: moved }), 48); +await install.scrape('https://www.amazon.com/s?k=stainless+tumbler', start + 7 * DAY + 3600000, cards(120, { offset: 280 }), 48); +console.log('synced', await install.flush()); +await terminate(db); +await deleteApp(app); +process.exit(0); From dc6b38df411888b80da99fa7750f7389a720bd2e Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:39:41 -0400 Subject: [PATCH 21/28] Document the shared schema, the checks that run the extension, and the e2e --- README.md | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b780c07..5a90947 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,11 @@ npm install # one-time npm run dev # landing page dev server npm run dev:dashboard # dashboard dev server, talks to the emulators npm run emulators # auth + firestore + hosting emulators (project demo-proscan) -npm run seed # demo data into the running emulators +npm run seed # fixture data (firebase-admin) into the running emulators +npm run seed:extension # data written by the extension's own sync module npm run build # clean dist/, then build landing + dashboard -npm test # typecheck, build, bundle check, rules tests +npm test # typecheck, schema check, unit tests, build, bundle check, rules tests +npm run test:e2e # extension sync seeds the emulator, the production bundle is checked npm run test:hosting # rewrites and cache headers (Linux or WSL only) npm run deploy # hosting + firestore rules + indexes to proscanbot ``` @@ -61,6 +63,31 @@ refuses a dirty tree or a branch other than `main`, so the stamp always names a real commit. CI (`.github/workflows/ci.yml`) runs the same checks on every pull request. +## The shared schema + +`packages/schema/index.js` is the extension's cloud schema, copied byte for +byte from `packages/schema/index.js` in the extension repo. It holds the +document shapes, validators and id builders for everything the extension +writes. `index.d.ts` next to it is ours. `npm run check:schema` fails when +the two copies differ; to update, copy the extension's file over ours and +adjust `index.d.ts`. + +The dashboard reads every document through a converter that runs the +schema's validator (`dashboard/src/lib/checked.ts`). A document that fails +is left out of the view and counted in a notice. `firestore.rules` checks +the same shapes on write. + +Several checks run the extension's real code from its checkout: the schema +check, the sync rules test, the e2e and `seed:extension`. They look for it +in `PROSCAN_EXT`, else `../ext` or `../AmazonSellerScraper`. Set +`PROSCAN_ALLOW_NO_EXT=1` to skip them where there is no checkout. CI checks +out the extension's `main`. + +`npm run test:e2e` builds the dashboard with `VITE_USE_EMULATOR=true` into +`dist-e2e/`, serves it with `vite preview` and drives it with Playwright. +`QA_CHROMIUM` points at an installed Chromium when Playwright's own is +missing. Screenshots and the exported workbook land in `.screenshots/e2e/`. + ## Revamp Adds sign-in, saving client/competitor data, and integration with the ProScan From cc3f64f1eff1027c076e30bee3376fca57bf1dfb Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:40:37 -0400 Subject: [PATCH 22/28] Make npm run seed write through the extension, and keep the old fixture as seed:fixture --- README.md | 6 +++--- package.json | 4 ++-- scripts/qa-e2e.mjs | 2 ++ scripts/seed-emulators.mjs | 5 +++++ scripts/seed-extension.mjs | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5a90947..5eabd3c 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ npm install # one-time npm run dev # landing page dev server npm run dev:dashboard # dashboard dev server, talks to the emulators npm run emulators # auth + firestore + hosting emulators (project demo-proscan) -npm run seed # fixture data (firebase-admin) into the running emulators -npm run seed:extension # data written by the extension's own sync module +npm run seed # scans written by the extension's own sync module +npm run seed:fixture # the older hand-made fixture (predates the schema) npm run build # clean dist/, then build landing + dashboard npm test # typecheck, schema check, unit tests, build, bundle check, rules tests npm run test:e2e # extension sync seeds the emulator, the production bundle is checked @@ -78,7 +78,7 @@ is left out of the view and counted in a notice. `firestore.rules` checks the same shapes on write. Several checks run the extension's real code from its checkout: the schema -check, the sync rules test, the e2e and `seed:extension`. They look for it +check, the sync rules test, the e2e and `npm run seed`. They look for it in `PROSCAN_EXT`, else `../ext` or `../AmazonSellerScraper`. Set `PROSCAN_ALLOW_NO_EXT=1` to skip them where there is no checkout. CI checks out the extension's `main`. diff --git a/package.json b/package.json index 6ed1cd1..d2632aa 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,8 @@ "deploy": "node scripts/deploy-check.mjs && npm run build && node scripts/check-bundle.mjs && firebase deploy --project proscanbot --only hosting,firestore:rules,firestore:indexes", "emulators": "node scripts/emulators.mjs start", "emulators:persist": "node scripts/emulators.mjs start --import=./.emulator-data --export-on-exit=./.emulator-data", - "seed": "node scripts/seed-emulators.mjs", - "seed:extension": "node scripts/seed-extension.mjs", + "seed": "node scripts/seed-extension.mjs", + "seed:fixture": "node scripts/seed-emulators.mjs", "test:rules": "node scripts/emulators.mjs exec \"node scripts/qa-sync-rules.mjs && node scripts/qa-lead-rules.mjs && node scripts/qa-board-count.mjs && node scripts/qa-adversarial-rules.mjs\"", "test:hosting": "node scripts/emulators.mjs exec --only hosting \"node scripts/qa-hosting.mjs\"", "test:e2e": "node scripts/emulators.mjs exec \"node scripts/e2e.mjs\"", diff --git a/scripts/qa-e2e.mjs b/scripts/qa-e2e.mjs index 3c42ff1..2ebff3a 100644 --- a/scripts/qa-e2e.mjs +++ b/scripts/qa-e2e.mjs @@ -1,5 +1,7 @@ // Throwaway integration-QA driver (Playwright, emulator-connected DEV server). // Usage: node scripts/qa-e2e.mjs +// Runs on the `npm run seed:fixture` data, which predates the shared +// schema; the dashboard now hides it. scripts/e2e.mjs replaces this. // - signs in as dev@proscan.test / proscan-dev // - walks every screen, screenshots to .screenshots/qa/ // - functional assertions: drawer URL contract, stage-change persistence, diff --git a/scripts/seed-emulators.mjs b/scripts/seed-emulators.mjs index 9020430..db5da5e 100644 --- a/scripts/seed-emulators.mjs +++ b/scripts/seed-emulators.mjs @@ -1,5 +1,10 @@ // ProScan emulator seed fixtures — see docs/architecture/data-model.md §4. // +// Predates the shared schema (packages/schema): these documents carry no +// `sv` and include Phase 5 fields, so the dashboard now leaves them out +// with a schema notice. `npm run seed` writes real extension output. +// Run this one with `npm run seed:fixture`. +// // SAFETY: this script may only ever talk to the local Firebase Emulator Suite. // It defaults the emulator host env vars and hard-fails if either is empty, so // writes against production Firestore/Auth are impossible by construction. diff --git a/scripts/seed-extension.mjs b/scripts/seed-extension.mjs index 1cfeaa2..ad4e27a 100644 --- a/scripts/seed-extension.mjs +++ b/scripts/seed-extension.mjs @@ -5,7 +5,7 @@ // extension writes. // // npm run emulators (one terminal) -// npm run seed:extension (another) +// npm run seed (another) process.env.FIRESTORE_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_FIRESTORE_PORT || 8080}`; process.env.FIREBASE_AUTH_EMULATOR_HOST ??= `127.0.0.1:${process.env.EMU_AUTH_PORT || 9099}`; From 276ac36dd7c6667b6a0f3207cb2a66ce466f69ca Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:47:03 -0400 Subject: [PATCH 23/28] Drop the UTC dayKey helper the run inbox no longer uses --- dashboard/src/features/runs/groupRuns.ts | 5 +++-- dashboard/src/lib/format.ts | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dashboard/src/features/runs/groupRuns.ts b/dashboard/src/features/runs/groupRuns.ts index 13a99ef..5460396 100644 --- a/dashboard/src/features/runs/groupRuns.ts +++ b/dashboard/src/features/runs/groupRuns.ts @@ -1,7 +1,8 @@ // Date-bucketing for the Run Inbox: Today / This week / Earlier. // Buckets key off the run's dayKey, the local date where it ran (falling -// back to startedAt), so the grouping matches the date on each card. Input arrives newest- -// first from runsNewestFirst(); bucketing preserves that order. +// back to startedAt), so the grouping matches the date on each card. +// Input arrives newest-first from runsNewestFirst(); bucketing keeps that +// order. import type { Run } from '../../lib/types'; import { dayKeyDiff, localDayKey, toMillis } from '../../lib/format'; diff --git a/dashboard/src/lib/format.ts b/dashboard/src/lib/format.ts index 1b6df48..d2ef40a 100644 --- a/dashboard/src/lib/format.ts +++ b/dashboard/src/lib/format.ts @@ -69,11 +69,6 @@ export function relativeTime(at: TimeLike): string { return future ? `in ${label}` : `${label} ago`; } -/** UTC 'YYYY-MM-DD' for a date (default: now). */ -export function dayKey(date: Date = new Date()): string { - return date.toISOString().slice(0, 10); -} - /** Local 'YYYY-MM-DD', the way the extension writes run and history dayKeys. */ export function localDayKey(date: Date = new Date()): string { return dayKeyOf(date.getTime(), date.getTimezoneOffset()); From a1bb939947278197e37028bdf933ebb06414726e Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 06:49:25 -0400 Subject: [PATCH 24/28] Ignore line endings when comparing the vendored schema --- scripts/check-schema.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/check-schema.mjs b/scripts/check-schema.mjs index e69d660..505cc4a 100644 --- a/scripts/check-schema.mjs +++ b/scripts/check-schema.mjs @@ -15,13 +15,15 @@ const check = (name, ok, detail = '') => { console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` (${detail})`}`); }; -const src = readFileSync(ours, 'utf8'); +// Line endings are ignored: autocrlf checks files out with CRLF on Windows. +const lf = (text) => text.replace(/\r\n/g, '\n'); +const src = lf(readFileSync(ours, 'utf8')); const exported = [...src.matchAll(/^export (?:const|function) (\w+)/gm)].map((m) => m[1]); const missing = exported.filter((name) => !new RegExp(`declare (?:const|function) ${name}\\b`).test(types)); check('index.d.ts declares every export', missing.length === 0, missing.join(', ')); const ext = requireExtension('the schema check'); -const theirs = readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8'); +const theirs = lf(readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8')); check(`index.js matches ${ext}`, theirs === src, 'copy the extension file over ours'); console.log(failed === 0 ? 'RESULT: PASS' : `RESULT: FAIL (${failed})`); From 35b2734ff7a86454f90353f938425635588cf632 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 16:16:27 -0400 Subject: [PATCH 25/28] Keep every loaded page under one live listener, and recount as it changes Load more now grows the listener instead of freezing a cursor page, so rows a sync pushes past a page boundary stay on screen. The header count recounts after live changes, never drops below what is loaded, and leaves out documents that failed the schema check. --- dashboard/src/lib/hooks.ts | 34 +++++--- dashboard/src/lib/paging.ts | 131 ++++++++++++++---------------- dashboard/src/routes/Products.tsx | 10 ++- dashboard/src/routes/Runs.tsx | 8 +- 4 files changed, 96 insertions(+), 87 deletions(-) diff --git a/dashboard/src/lib/hooks.ts b/dashboard/src/lib/hooks.ts index b73d344..9ab18bf 100644 --- a/dashboard/src/lib/hooks.ts +++ b/dashboard/src/lib/hooks.ts @@ -10,6 +10,7 @@ import { createElement, useContext, useEffect, + useRef, useState, useSyncExternalStore, type DependencyList, @@ -165,29 +166,42 @@ export function useSnapshotQuery( /* ── server-side count (aggregate, no documents read) ───────────────── */ /** getCountFromServer for a query. `null` while unknown or when the count - * failed; callers fall back to "showing first N". Re-runs on dep change. */ + * failed; callers fall back to "showing first N". Resets on dep change. + * A change of `refresh` (say, a live list's snapshot counter) recounts + * after a short pause and keeps the old number until the new one lands. */ export function useServerCount( queryFactory: () => Query | null, deps: DependencyList, + refresh: unknown = 0, ): number | null { const [count, setCount] = useState(null); + const fresh = useRef(true); useEffect(() => { - const q = queryFactory(); + fresh.current = true; setCount(null); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps); + useEffect(() => { + const q = queryFactory(); if (!q) return; let cancelled = false; - getCountFromServer(q) - .then((snap) => { - if (!cancelled) setCount(snap.data().count); - }) - .catch((error: unknown) => { - console.error('[proscan] count query failed', error); - }); + const delay = fresh.current ? 0 : 1000; + fresh.current = false; + const timer = setTimeout(() => { + getCountFromServer(q) + .then((snap) => { + if (!cancelled) setCount(snap.data().count); + }) + .catch((error: unknown) => { + console.error('[proscan] count query failed', error); + }); + }, delay); return () => { cancelled = true; + clearTimeout(timer); }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, deps); + }, [...deps, refresh]); return count; } diff --git a/dashboard/src/lib/paging.ts b/dashboard/src/lib/paging.ts index 98501c4..c374476 100644 --- a/dashboard/src/lib/paging.ts +++ b/dashboard/src/lib/paging.ts @@ -1,10 +1,11 @@ -// Cursor pagination over an ordered query. The first page is a live -// listener, so a scan that is syncing shows up without a reload; each -// "load more" is a one-shot getDocs that starts after the last document -// already loaded. Nothing is ever capped silently: `hasMore` says when -// the server has more. +// Pagination over an ordered query. One live listener covers every page +// loaded so far (limit(size * pages)), so a scan that is syncing shows up +// without a reload and a row pushed down past a page boundary is never +// dropped between pages. "Load more" grows the listener by one page, which +// re-reads what is already loaded; that is the price of never losing rows. +// Nothing is ever capped silently: `hasMore` says when the server has more. -import { useCallback, useEffect, useMemo, useRef, useState, type DependencyList } from 'react'; +import { useEffect, useState, type DependencyList } from 'react'; import { getDocs, limit, @@ -15,7 +16,6 @@ import { type QueryDocumentSnapshot, } from 'firebase/firestore'; import { splitChecked, type SchemaProblem } from './checked'; -import { reportError } from './errors'; export const PAGE_SIZE = 200; @@ -27,6 +27,8 @@ export interface PagedState { hasMore: boolean; loadingMore: boolean; loadMore: () => void; + /** Bumps on every server snapshot that changed documents. */ + changes: number; } interface Page { @@ -41,100 +43,87 @@ function toPage(docs: QueryDocumentSnapshot[], size: number): Page { return { rows: valid, invalid, last: docs[docs.length - 1] ?? null, full: docs.length >= size }; } -/** - * Pages through `queryFactory()` (ordered, no limit) `size` documents at a - * time. `keyOf` dedupes a row that moved from one page into another while - * the live first page changed. - */ +const sameDeps = (a: DependencyList, b: DependencyList) => + a.length === b.length && a.every((v, i) => Object.is(v, b[i])); + +interface Shown extends Page { + deps: DependencyList; + pages: number; +} + +/** Pages through `queryFactory()` (ordered, no limit) `size` documents at a time. */ export function usePagedQuery( queryFactory: () => Query | null, deps: DependencyList, - keyOf: (row: T) => string, size: number = PAGE_SIZE, ): PagedState { - const [first, setFirst] = useState | null>(null); - const [more, setMore] = useState[]>([]); - const [error, setError] = useState(null); - const [loadingMore, setLoadingMore] = useState(false); + // Pages wanted for these deps; a deps change starts again at one page. + const [grow, setGrow] = useState({ deps, pages: 1 }); + const pages = sameDeps(grow.deps, deps) ? grow.pages : 1; + const [shownRaw, setShown] = useState | null>(null); + const shown = shownRaw && sameDeps(shownRaw.deps, deps) ? shownRaw : null; + const [error, setError] = useState<{ deps: DependencyList; err: Error } | null>(null); + const [changes, setChanges] = useState(0); const [active, setActive] = useState(true); - const base = useRef | null>(null); - const generation = useRef(0); useEffect(() => { const q = queryFactory(); - base.current = q; - generation.current++; - setFirst(null); - setMore([]); setError(null); - setLoadingMore(false); setActive(q !== null); if (!q) return; + const want = size * pages; + let fromServer = false; // Metadata changes too, so the server's answer arrives even when it - // matches an empty cache (F-48). + // matches an empty cache (F-48). A grown listener also skips its cache + // answer, which would briefly show fewer rows than were loaded. return onSnapshot( - query(q, limit(size)), + query(q, limit(want)), { includeMetadataChanges: true }, (snap) => { - if (snap.empty && snap.metadata.fromCache) return; - setFirst(toPage(snap.docs, size)); + const cached = snap.metadata.fromCache; + if (cached && !fromServer && (snap.empty || pages > 1)) return; + if (!cached) fromServer = true; + setShown({ ...toPage(snap.docs, want), deps, pages }); + if (!cached && snap.docChanges().length > 0) setChanges((n) => n + 1); }, (err) => { - console.error('[proscan] first page failed', err); - setError(err); + console.error('[proscan] paged query failed', err); + setError({ deps, err }); }, ); // eslint-disable-next-line react-hooks/exhaustive-deps - }, deps); - - const lastPage = more.length > 0 ? more[more.length - 1] : first; - const hasMore = !!lastPage && lastPage.full; - - const loadMore = useCallback(() => { - const q = base.current; - if (!q || !lastPage?.last || loadingMore) return; - const gen = generation.current; - setLoadingMore(true); - getDocs(query(q, startAfter(lastPage.last), limit(size))) - .then((snap) => { - if (gen !== generation.current) return; - setMore((pages) => [...pages, toPage(snap.docs, size)]); - }) - .catch((err: unknown) => { - if (gen === generation.current) reportError('load more products', err); - }) - .finally(() => { - if (gen === generation.current) setLoadingMore(false); - }); - }, [lastPage, loadingMore, size]); + }, [...deps, pages, size]); - const { data, invalid } = useMemo(() => { - const pages = first ? [first, ...more] : []; - const seen = new Set(); - const rows: T[] = []; - for (const page of pages) { - for (const row of page.rows) { - const k = keyOf(row); - if (seen.has(k)) continue; - seen.add(k); - rows.push(row); - } - } - return { data: rows, invalid: pages.flatMap((p) => p.invalid) }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [first, more]); + const hasMore = !!shown && shown.full; + const loadingMore = !!shown && shown.pages < pages; + const loadMore = () => { + if (hasMore && !loadingMore) setGrow({ deps, pages: pages + 1 }); + }; + const err = error && sameDeps(error.deps, deps) ? error.err : null; return { - data, - invalid, - loading: !error && active && first === null, - error, + data: shown?.rows ?? [], + invalid: shown?.invalid ?? [], + loading: !err && active && shown === null, + error: err, hasMore, loadingMore, loadMore, + changes, }; } +/** The count to show for a paged list. When everything is loaded that is + * the loaded rows (invalid documents are reported apart). Otherwise it is + * the server count minus the invalid documents seen, and never below what + * is already on screen, since the count can lag a live sync. */ +export function pagedTotal(state: PagedState, serverCount: number | null): number | null { + if (state.loading) return serverCount; + if (!state.hasMore) return state.data.length; + if (serverCount === null) return null; + return Math.max(serverCount - state.invalid.length, state.data.length); +} + /** Every document of `q`, read `size` at a time with a cursor. For export. */ export async function fetchAll( q: Query, diff --git a/dashboard/src/routes/Products.tsx b/dashboard/src/routes/Products.tsx index f92ebc3..c6fbfa4 100644 --- a/dashboard/src/routes/Products.tsx +++ b/dashboard/src/routes/Products.tsx @@ -22,7 +22,7 @@ import { type SortingState, } from '@tanstack/react-table'; import { useDocOnce, useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; -import { fetchAll, usePagedQuery, PAGE_SIZE } from '../lib/paging'; +import { fetchAll, pagedTotal, usePagedQuery, PAGE_SIZE } from '../lib/paging'; import { reportError } from '../lib/errors'; import { productRef, @@ -91,9 +91,13 @@ export default function Products() { const latest = usePagedQuery( () => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId], - (p) => p.asin, ); - const total = useServerCount(() => (view === 'latest' ? scopeQuery() : null), [wid, view, sourceId]); + const serverCount = useServerCount( + () => (view === 'latest' ? scopeQuery() : null), + [wid, view, sourceId], + latest.changes, + ); + const total = pagedTotal(latest, serverCount); const sourcesState = useSnapshotQuery( () => (wid ? sourcesQuery(wid) : null), diff --git a/dashboard/src/routes/Runs.tsx b/dashboard/src/routes/Runs.tsx index 4430f95..499af62 100644 --- a/dashboard/src/routes/Runs.tsx +++ b/dashboard/src/routes/Runs.tsx @@ -7,7 +7,7 @@ import { Fragment, useMemo } from 'react'; import { useNavigate } from 'react-router-dom'; import { useServerCount, useSnapshotQuery, useWorkspace } from '../lib/hooks'; -import { usePagedQuery } from '../lib/paging'; +import { pagedTotal, usePagedQuery } from '../lib/paging'; import { runsNewestFirst, sources } from '../lib/queries'; import type { Source } from '../lib/types'; import PageHeader from '../components/PageHeader'; @@ -30,10 +30,12 @@ export default function Runs() { const runsState = usePagedQuery( () => (wid ? runsNewestFirst(wid) : null), [wid], - (r) => r.runId, RUNS_PAGE, ); - const total = useServerCount(() => (wid ? runsNewestFirst(wid) : null), [wid]); + const total = pagedTotal( + runsState, + useServerCount(() => (wid ? runsNewestFirst(wid) : null), [wid], runsState.changes), + ); const sourcesState = useSnapshotQuery( () => (wid ? sources(wid) : null), [wid], From 14a8f732a14645e3c54ef2f5a629def4a720e387 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 16:16:53 -0400 Subject: [PATCH 26/28] Keep check:schema's exit code when the extension checkout is missing PROSCAN_ALLOW_NO_EXT=1 used to exit 0 from inside requireExtension, after the index.d.ts check had already failed. --- scripts/check-schema.mjs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/check-schema.mjs b/scripts/check-schema.mjs index 505cc4a..8c35361 100644 --- a/scripts/check-schema.mjs +++ b/scripts/check-schema.mjs @@ -5,7 +5,7 @@ import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; -import { WEB_ROOT, requireExtension } from './lib/extension.mjs'; +import { WEB_ROOT, findExtension } from './lib/extension.mjs'; const ours = resolve(WEB_ROOT, 'packages', 'schema', 'index.js'); const types = readFileSync(resolve(WEB_ROOT, 'packages', 'schema', 'index.d.ts'), 'utf8'); @@ -22,9 +22,17 @@ const exported = [...src.matchAll(/^export (?:const|function) (\w+)/gm)].map((m) const missing = exported.filter((name) => !new RegExp(`declare (?:const|function) ${name}\\b`).test(types)); check('index.d.ts declares every export', missing.length === 0, missing.join(', ')); -const ext = requireExtension('the schema check'); -const theirs = lf(readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8')); -check(`index.js matches ${ext}`, theirs === src, 'copy the extension file over ours'); +// Without the extension only the byte comparison is skipped; the check +// above still decides the exit code. +const ext = findExtension(); +if (ext) { + const theirs = lf(readFileSync(resolve(ext, 'packages', 'schema', 'index.js'), 'utf8')); + check(`index.js matches ${ext}`, theirs === src, 'copy the extension file over ours'); +} else if (process.env.PROSCAN_ALLOW_NO_EXT === '1') { + console.log(' SKIP index.js matches the extension: no extension checkout (PROSCAN_ALLOW_NO_EXT=1)'); +} else { + check('extension checkout found', false, 'set PROSCAN_EXT, or PROSCAN_ALLOW_NO_EXT=1 to skip'); +} console.log(failed === 0 ? 'RESULT: PASS' : `RESULT: FAIL (${failed})`); process.exit(failed === 0 ? 0 : 1); From 892865f904b92fd99d7a6e1bba0e5c7a30382c21 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 16:19:47 -0400 Subject: [PATCH 27/28] Serve the favicon in the e2e, and stop its preview server when launch fails vite preview of the dashboard alone 404s /icon128.png, which failed the console check. A failed chromium launch left vite running, and a later run then reused it; preview now refuses a port that already answers. --- scripts/e2e.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs index d75aeda..d218e63 100644 --- a/scripts/e2e.mjs +++ b/scripts/e2e.mjs @@ -124,6 +124,9 @@ function build() { } async function preview() { + // A server left over from an earlier run would stand in for this build. + const taken = await fetch(BASE).then(() => true, () => false); + if (taken) throw new Error(`port ${PREVIEW_PORT} already answers; stop that server or set E2E_PREVIEW_PORT`); const child = spawn( 'npx', ['vite', 'preview', '--config', 'vite.dashboard.config.ts', '--outDir', OUT_DIR, '--port', String(PREVIEW_PORT), '--strictPort'], @@ -131,6 +134,7 @@ async function preview() { { cwd: WEB_ROOT, shell: true, stdio: 'ignore', detached: process.platform !== 'win32' }, ); for (let i = 0; i < 60; i++) { + if (child.exitCode !== null) throw new Error(`vite preview exited with ${child.exitCode}`); try { const res = await fetch(BASE); if (res.ok) return child; @@ -164,6 +168,11 @@ async function newPage(browser, consoleErrors) { await ctx.route(/^https?:\/\/([^/]*\.)?(amazon\.com|media-amazon\.com)\//, (route) => route.fulfill({ status: 200, contentType: 'image/png', body: PNG }), ); + // Hosting serves public/ at the root, which vite preview of the dashboard + // alone does not; without this the favicon 404s into the console check. + await ctx.route(`http://localhost:${PREVIEW_PORT}/icon128.png`, (route) => + route.fulfill({ status: 200, contentType: 'image/png', path: resolve(WEB_ROOT, 'public', 'icon128.png') }), + ); const page = await ctx.newPage(); page.on('console', (msg) => { if (msg.type() === 'error') consoleErrors.push(msg.text()); @@ -238,9 +247,10 @@ async function main() { console.log('[build] production bundle against the emulators'); build(); const server = await preview(); - const browser = await chromium.launch({ executablePath: process.env.QA_CHROMIUM || undefined }); const consoleErrors = []; + let browser; try { + browser = await chromium.launch({ executablePath: process.env.QA_CHROMIUM || undefined }); const { ctx, page } = await newPage(browser, consoleErrors); console.log('[board] all sources, latest'); @@ -324,7 +334,7 @@ async function main() { // the dashboard falls back to updatePassword; nothing else may log. consoleErrors.push(...googleErrors.filter((t) => !/status of 400/.test(t))); } finally { - await browser.close().catch(() => {}); + await browser?.close().catch(() => {}); kill(server); } From 632ca5eac7525a71ae7b3d596fa163ddcdb797a5 Mon Sep 17 00:00:00 2001 From: Enes Yilmaz Date: Thu, 24 Sep 2026 17:10:17 -0400 Subject: [PATCH 28/28] Pass deleteField to the extension sync in the e2e seed The extension's sync now needs it to keep every source a product was seen in, and that fix also clears the NEW-SYNC-1 known failure. --- scripts/e2e.mjs | 3 +-- scripts/lib/extension-sync.mjs | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs index d218e63..1cc47d6 100644 --- a/scripts/e2e.mjs +++ b/scripts/e2e.mjs @@ -292,8 +292,7 @@ async function main() { console.log('[board] one source: deltas against its previous run'); await page.selectOption('.board-select', 's_A3K9XELT4QZ6M2'); sub = await waitSubtitle(page, /deltas against the run of/); - // The 50 ASINs the keyword run also saw lose the storefront from sourceIds. - knownFailure('NEW-SYNC-1 (F-20 area): storefront scope counts all its products', sub.includes(`200 of ${fmt(STORE_N)} products loaded`), sub); + check('storefront scope counts all its products', sub.includes(`200 of ${fmt(STORE_N)} products loaded`), sub); await checkColumns(page, 'storefront', { deltas: 0.5 }); await page.screenshot({ path: resolve(SHOTS, 'board-storefront.png') }); diff --git a/scripts/lib/extension-sync.mjs b/scripts/lib/extension-sync.mjs index db5e0a2..97bf538 100644 --- a/scripts/lib/extension-sync.mjs +++ b/scripts/lib/extension-sync.mjs @@ -15,10 +15,11 @@ import { writeBatch, Timestamp, arrayUnion, + deleteField, FieldPath, } from 'firebase/firestore'; -const FIRESTORE = { doc, getDoc, writeBatch, Timestamp, arrayUnion, FieldPath }; +const FIRESTORE = { doc, getDoc, writeBatch, Timestamp, arrayUnion, deleteField, FieldPath }; const quiet = { log() {}, warn() {}, error() {} }; /** Loads the engine, db, sync and schema modules from an extension checkout. */