diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..48acdf6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - name: Jest + run: npx jest + + - name: Tool tests + run: npm run test:tools + + - name: Build + run: npm run build + + - name: Permission lock and version gate + run: npm run lock + + - name: Secret scan + run: npm run scan:secrets + + - name: Store zip gates + run: node tools/zip.mjs + + - uses: actions/upload-artifact@v4 + with: + name: store-zip + path: dist-zips/*.zip + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 076aaa6..8dfb469 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ coverage/ dist/ build/ *.zip +dist-zips/ diff --git a/README.md b/README.md index f72a155..eb0bd6d 100644 --- a/README.md +++ b/README.md @@ -125,14 +125,22 @@ See [docs/PRICE_SPREAD_ANALYSIS.md](docs/PRICE_SPREAD_ANALYSIS.md) for the full ## Installation -1. Clone the repository: +1. Clone the repository and build it: ```bash - git clone https://github.com/enesyilmaz7/AmazonSellerScraper.git + git clone https://github.com/EnesYilmazcode/AmazonSellerScraper.git + cd AmazonSellerScraper + npm ci + npm run build ``` 2. Open Chrome and navigate to `chrome://extensions/` 3. Enable **Developer mode** (top-right toggle) -4. Click **Load unpacked** and select the project folder -5. Click the ProScan extension icon → **Settings** → paste your [Gemini API key](https://aistudio.google.com/apikey) (free) +4. Click **Load unpacked** and select the `dist/` folder. The repo root does not load on its own, because the service worker has to be bundled. + +For local Firebase work, `npm run build:dev` points the build at the emulators under the `demo-proscan` project. + +## Release checks + +`npm test` runs the Jest suite and the tool tests. `npm run check` builds, then runs the permission lock (nothing may be added over `tools/live-manifest.json`, the published v2.0 manifest), the version gate and the secret scan. `npm run zip` writes the store package to `dist-zips/` and refuses a dev build, a stray file, uncommitted changes (`node tools/zip.mjs --allow-dirty` overrides that for local tries), or any gate failure. CI runs all of these. ## Usage diff --git a/manifest.json b/manifest.json index 0c2f13b..40894c9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,59 +1,52 @@ -{ - "manifest_version": 3, - "name": "ProScan - Amazon Product Scraper", - "version": "2.0", - "description": "Scrape Amazon seller products with analytics and AI-powered product insights", - "permissions": [ - "storage", - "downloads" - ], - "host_permissions": [ - "*://*.amazon.com/*", - "https://generativelanguage.googleapis.com/*", - "https://identitytoolkit.googleapis.com/*", - "https://securetoken.googleapis.com/*", - "https://firestore.googleapis.com/*", - "https://www.googleapis.com/*", - "https://*.firebaseapp.com/*", - "http://127.0.0.1/*", - "http://localhost/*" - ], - "action": { - "default_popup": "popup/popup.html", - "default_icon": { - "16": "assets/icons/icon16.png", - "48": "assets/icons/icon48.png", - "128": "assets/icons/icon128.png" - } - }, - "icons": { - "16": "assets/icons/icon16.png", - "48": "assets/icons/icon48.png", - "128": "assets/icons/icon128.png" - }, - "content_scripts": [ - { - "matches": ["*://*.amazon.com/*"], - "js": [ - "scripts/modules/price.js", - "scripts/modules/delta.js", - "scripts/content/scraper.js", - "scripts/content/chatbot.js", - "scripts/content/offer-fetcher.js" - ], - "run_at": "document_idle" - } - ], - "content_security_policy": { - "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self'; connect-src 'self' https://identitytoolkit.googleapis.com https://securetoken.googleapis.com https://firestore.googleapis.com https://www.googleapis.com https://*.firebaseapp.com http://127.0.0.1:* http://localhost:*" - }, - "background": { - "service_worker": "scripts/background/service-worker.js" - }, - "web_accessible_resources": [ - { - "resources": ["styles/chatbot.css", "libs/*.js", "assets/icons/*.png"], - "matches": ["*://*.amazon.com/*"] - } - ] -} +{ + "manifest_version": 3, + "name": "ProScan - Amazon Product Scraper", + "version": "2.1.0", + "description": "Scrape Amazon seller products with analytics and AI-powered product insights", + "permissions": [ + "storage", + "downloads" + ], + "host_permissions": [ + "*://*.amazon.com/*", + "https://generativelanguage.googleapis.com/*" + ], + "action": { + "default_popup": "popup/popup.html", + "default_icon": { + "16": "assets/icons/icon16.png", + "48": "assets/icons/icon48.png", + "128": "assets/icons/icon128.png" + } + }, + "icons": { + "16": "assets/icons/icon16.png", + "48": "assets/icons/icon48.png", + "128": "assets/icons/icon128.png" + }, + "content_scripts": [ + { + "matches": ["*://*.amazon.com/*"], + "js": [ + "scripts/modules/price.js", + "scripts/modules/delta.js", + "scripts/content/scraper.js", + "scripts/content/chatbot.js", + "scripts/content/offer-fetcher.js" + ], + "run_at": "document_idle" + } + ], + "content_security_policy": { + "extension_pages": "script-src 'self'; object-src 'self'; connect-src 'self' https://identitytoolkit.googleapis.com https://securetoken.googleapis.com https://firestore.googleapis.com https://generativelanguage.googleapis.com" + }, + "background": { + "service_worker": "scripts/background/service-worker.js" + }, + "web_accessible_resources": [ + { + "resources": ["styles/chatbot.css", "libs/*.js", "assets/icons/*.png"], + "matches": ["*://*.amazon.com/*"] + } + ] +} diff --git a/package.json b/package.json index b2638c9..3b2be96 100644 --- a/package.json +++ b/package.json @@ -3,11 +3,16 @@ "private": true, "scripts": { "build": "node tools/build.mjs", + "build:dev": "node tools/build.mjs --dev", "zip": "node tools/build.mjs && node tools/zip.mjs", - "test": "jest --verbose", + "test": "jest --verbose && npm run test:tools", "test:unit": "jest tests/unit --verbose", "test:integration": "jest tests/integration --verbose", - "test:coverage": "jest --coverage" + "test:tools": "node --test --test-concurrency=1 \"tools/tests/*.test.mjs\"", + "test:coverage": "jest --coverage", + "lock": "node tools/permission-lock.mjs && node tools/version-gate.mjs", + "scan:secrets": "node tools/secret-scan.mjs", + "check": "npm run build && npm run lock && npm run scan:secrets" }, "devDependencies": { "adm-zip": "^0.5.17", diff --git a/scripts/background/firebase-config.js b/scripts/background/firebase-config.js index ebd3912..a001848 100644 --- a/scripts/background/firebase-config.js +++ b/scripts/background/firebase-config.js @@ -1,15 +1,21 @@ /** * @fileoverview Firebase config for the ProScan extension. * - * Same Firebase project (proscanbot) for dev and prod — only the emulator - * wiring differs, gated by PROSCAN_ENV which esbuild inlines at build time - * (see tools/build.mjs `define`). Web API keys are public identifiers, not - * secrets; access is governed entirely by Firestore security rules. + * Prod talks to the proscanbot project. The dev build + * (`PROSCAN_ENV=dev npm run build`) sets the literal __PROSCAN_EMULATOR__ + * through esbuild `define` and points at the local emulator suite under the + * demo-proscan project, the same id the dashboard dev build uses. Web API + * keys are public identifiers, not secrets; access is governed by the + * Firestore security rules. * * @module FirebaseConfig */ -export const FIREBASE_CONFIG = { +/* global __PROSCAN_EMULATOR__ */ +export const USE_EMULATOR = + typeof __PROSCAN_EMULATOR__ !== 'undefined' && __PROSCAN_EMULATOR__ === true; + +const PROD_CONFIG = { apiKey: 'AIzaSyAp0HrcvFwpMxrlqbxa9xjUvwGoTa7QpUU', authDomain: 'proscanbot.firebaseapp.com', projectId: 'proscanbot', @@ -18,10 +24,17 @@ export const FIREBASE_CONFIG = { messagingSenderId: '886322190589', }; -// Build dev with `PROSCAN_ENV=dev npm run build` to point at the local emulator. -export const USE_EMULATOR = - typeof process !== 'undefined' && - process.env && - process.env.PROSCAN_ENV === 'dev'; +// demo-* projects only ever talk to the emulators. Written inline so the +// prod build folds them away. +export const FIREBASE_CONFIG = USE_EMULATOR + ? { + ...PROD_CONFIG, + authDomain: 'demo-proscan.firebaseapp.com', + projectId: 'demo-proscan', + storageBucket: 'demo-proscan.appspot.com', + } + : PROD_CONFIG; -export const EMULATOR = { host: '127.0.0.1', authPort: 9099, firestorePort: 8080 }; +export const EMULATOR = USE_EMULATOR + ? { host: '127.0.0.1', authPort: 9099, firestorePort: 8080 } + : null; diff --git a/scripts/background/service-worker.js b/scripts/background/service-worker.js index affda8e..303e792 100644 --- a/scripts/background/service-worker.js +++ b/scripts/background/service-worker.js @@ -23,21 +23,6 @@ import { syncToCloud } from './sync.js'; /** @const {string} Gemini API endpoint for content generation */ const GEMINI_API_URL = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent'; -/** - * Fallback API key (base64-encoded). Used only when the user hasn't - * configured their own key via the popup settings panel. - * @const {string} - * @private - */ -const _t = 'QUl6YVN5RHdfOVhQLXRpQ0tLX3lkQThCd0ZrZUpxNWdTdTAxNUhj'; - -/** - * Decode the fallback API key. - * @returns {string} Decoded API key - * @private - */ -const _dk = () => atob(_t); - /** * Main message listener -- routes messages between extension components. * @@ -74,17 +59,19 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { * Handle a chat message by calling the Gemini 2.0 Flash API. * * Builds a prompt with the system role, product context, and user question. - * Uses the user's API key from chrome.storage if available, otherwise - * falls back to the built-in key. + * Uses the user's API key from chrome.storage. There is no built-in key. * * @param {string} question - User's natural language question * @param {Object[]} products - Array of product objects for context * @returns {Promise} AI-generated response text - * @throws {Error} On invalid API key or Gemini API failure + * @throws {Error} On a missing or invalid API key, or Gemini API failure */ async function handleChatMessage(question, products) { const data = await chrome.storage.local.get(['geminiApiKey']); - const apiKey = data.geminiApiKey || _dk(); + const apiKey = data.geminiApiKey; + if (!apiKey) { + throw new Error('AI chat needs a Gemini API key, and none is set.'); + } const productCount = products.length; const productList = products.map(p => diff --git a/tools/build.mjs b/tools/build.mjs index 6657508..2828c4e 100644 --- a/tools/build.mjs +++ b/tools/build.mjs @@ -1,14 +1,14 @@ // tools/build.mjs — builds the extension into dist/. // -// - Bundles exactly one entry (the MV3 service worker) with esbuild. Today the -// worker has no imports, so output is semantically identical to the source; -// this establishes the M3 path for firebase/* npm imports. -// - Copies every other shipped file VERBATIM, preserving paths, so -// manifest.json needs zero rewriting. +// - Bundles exactly one entry (the MV3 service worker, with firebase/*) with +// esbuild. +// - Copies every other shipped file verbatim, preserving paths. manifest.json +// is the prod manifest; `PROSCAN_ENV=dev` adds the emulator origins to +// connect-src (see manifestForEnv) and points Firebase at demo-proscan. // - Popup files and scripts/modules/* are deliberately NOT bundled: // popup.html loads them as plain ' + ); + assert.deepEqual(htmlReferencesMissing(dir), ['popup/popup.html -> ../gone.js']); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/tests/permission-lock.test.mjs b/tools/tests/permission-lock.test.mjs new file mode 100644 index 0000000..1411b87 --- /dev/null +++ b/tools/tests/permission-lock.test.mjs @@ -0,0 +1,88 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { checkPermissionLock, LIVE_MANIFEST } from '../permission-lock.mjs'; + +const live = JSON.parse(fs.readFileSync(LIVE_MANIFEST, 'utf8')); +const clone = () => structuredClone(live); + +test('the live manifest passes against itself', () => { + assert.deepEqual(checkPermissionLock(live, live), []); +}); + +test('removing permissions and hosts passes', () => { + const m = clone(); + m.permissions = ['storage']; + m.host_permissions = ['*://*.amazon.com/*']; + delete m.web_accessible_resources; + assert.deepEqual(checkPermissionLock(m, live), []); +}); + +test('an added host permission fails', () => { + const m = clone(); + m.host_permissions.push('https://firestore.googleapis.com/*'); + assert.match(checkPermissionLock(m, live).join('\n'), /host_permissions adds .*firestore/); +}); + +test('an added API permission fails', () => { + const m = clone(); + m.permissions.push('alarms'); + assert.equal(checkPermissionLock(m, live).length, 1); +}); + +test('optional permissions and optional hosts fail', () => { + const m = clone(); + m.optional_permissions = ['tabs']; + m.optional_host_permissions = ['']; + assert.equal(checkPermissionLock(m, live).length, 2); +}); + +test('a widened content script match fails', () => { + const m = clone(); + m.content_scripts[0].matches = ['*://*/*']; + assert.match(checkPermissionLock(m, live).join('\n'), /content_scripts adds/); +}); + +test('all_frames on a content script fails', () => { + const m = clone(); + m.content_scripts[0].all_frames = true; + assert.equal(checkPermissionLock(m, live).length, 1); +}); + +test('a widened web_accessible_resources match fails', () => { + const m = clone(); + m.web_accessible_resources[0].matches.push(''); + assert.equal(checkPermissionLock(m, live).length, 1); +}); + +test('a new externally_connectable key fails', () => { + const m = clone(); + m.externally_connectable = { matches: ['https://proscanbot.web.app/*'] }; + assert.match(checkPermissionLock(m, live).join('\n'), /externally_connectable/); +}); + +test('CSP changes are not permissions and pass', () => { + const m = clone(); + m.content_security_policy.extension_pages += "; connect-src 'self' https://firestore.googleapis.com"; + assert.deepEqual(checkPermissionLock(m, live), []); +}); + +test('any new top-level key outside the safe list fails', () => { + for (const [key, value] of [ + ['chrome_settings_overrides', { search_provider: { name: 'x', keyword: 'x', search_url: 'https://x.test/?q={searchTerms}', favicon_url: 'https://x.test/f.ico', encoding: 'UTF-8', is_default: true } }], + ['automation', { desktop: true }], + ['devtools_page', 'devtools.html'], + ['sandbox', { pages: ['sandbox.html'] }], + ]) { + const m = clone(); + m[key] = value; + assert.match(checkPermissionLock(m, live).join(' | '), new RegExp(`new manifest key "${key}"`), key); + } +}); + +test('safe new keys such as minimum_chrome_version pass', () => { + const m = clone(); + m.minimum_chrome_version = '116'; + m.version_name = '2.1 beta'; + assert.deepEqual(checkPermissionLock(m, live), []); +}); diff --git a/tools/tests/secret-scan.test.mjs b/tools/tests/secret-scan.test.mjs new file mode 100644 index 0000000..2011943 --- /dev/null +++ b/tools/tests/secret-scan.test.mjs @@ -0,0 +1,43 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { scanText, ALLOWED } from '../secret-scan.mjs'; + +// Built at runtime so this file does not trip the scan itself. +const fakeKey = 'AI' + 'za' + 'Sy' + 'Q'.repeat(33); + +test('finds a plain Google API key', () => { + const hits = scanText(`const k = '${fakeKey}';`); + assert.equal(hits.length, 1); + assert.equal(hits[0].name, 'Google API key'); +}); + +test('finds a base64-encoded key, as the old Gemini fallback was', () => { + const b64 = Buffer.from(fakeKey).toString('base64'); + const hits = scanText(`const _t = '${b64}';`); + assert.equal(hits.length, 1); + assert.match(hits[0].name, /base64-encoded/); +}); + +test('finds a base64 key at an odd offset inside a longer blob', () => { + const b64 = Buffer.from('xy' + fakeKey + 'tail').toString('base64'); + assert.equal(scanText(b64).length, 1); +}); + +test('reports the line number', () => { + assert.equal(scanText(`a\nb\n${fakeKey}\n`)[0].line, 3); +}); + +test('allows the public Firebase web config key', () => { + const [key] = ALLOWED; + assert.deepEqual(scanText(`apiKey: '${key}'`), []); +}); + +test('finds private keys', () => { + const pem = '-----BEGIN ' + 'PRIVATE KEY-----\nabc\n'; + assert.equal(scanText(pem).length, 1); +}); + +test('ignores ordinary long identifiers and hashes', () => { + const text = 'sha512-' + 'a1b2c3d4'.repeat(11) + '\nconst someVeryLongIdentifierNameThatKeepsGoingAndGoing = 1;'; + assert.deepEqual(scanText(text), []); +}); diff --git a/tools/tests/version-gate.test.mjs b/tools/tests/version-gate.test.mjs new file mode 100644 index 0000000..91dc6d2 --- /dev/null +++ b/tools/tests/version-gate.test.mjs @@ -0,0 +1,23 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { compareVersions, parseVersion } from '../version-gate.mjs'; + +test('compares dotted versions numerically', () => { + assert.equal(compareVersions('2.1.0', '2.0'), 1); + assert.equal(compareVersions('2.0.0', '2.0'), 0); + assert.equal(compareVersions('2.10', '2.9'), 1); + assert.equal(compareVersions('1.9.9.9', '2.0'), -1); +}); + +test('rejects versions Chrome would reject', () => { + for (const v of ['2.1.0-beta', '02.1', '1.2.3.4.5', '70000', '', 'v2']) { + assert.throws(() => parseVersion(v), /invalid/); + } +}); + +test('the source manifest is above the live version', () => { + const src = JSON.parse(fs.readFileSync(new URL('../../manifest.json', import.meta.url))); + const live = JSON.parse(fs.readFileSync(new URL('../live-manifest.json', import.meta.url))); + assert.equal(compareVersions(src.version, live.version), 1); +}); diff --git a/tools/tests/zip.test.mjs b/tools/tests/zip.test.mjs new file mode 100644 index 0000000..af526be --- /dev/null +++ b/tools/tests/zip.test.mjs @@ -0,0 +1,74 @@ +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { buildExtension } from '../build.mjs'; +import { zipGateProblems } from '../zip.mjs'; + +let dir; +before(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'proscan-zip-')); +}); +after(() => fs.rmSync(dir, { recursive: true, force: true })); + +const fresh = (env = 'prod') => buildExtension({ env, outDir: dir }); + +test('a clean prod build passes every gate', async () => { + await fresh(); + assert.deepEqual(zipGateProblems(dir), {}); +}); + +test('stray zips, markdown and server files are rejected', async () => { + await fresh(); + fs.writeFileSync(path.join(dir, 'proscan-v2.0.zip'), 'x'); + fs.writeFileSync(path.join(dir, 'CLAUDE.md'), 'x'); + fs.mkdirSync(path.join(dir, 'server')); + fs.writeFileSync(path.join(dir, 'server/app.py'), 'x'); + const p = zipGateProblems(dir); + assert.equal(p.ALLOWLIST.length, 3); + assert.equal(p['LEGACY TRIPWIRE'].length, 3); +}); + +test('a dev build is refused', async () => { + await fresh('dev'); + assert.ok(zipGateProblems(dir)['PROD ONLY']); +}); + +test('an escalated or unbumped manifest is refused', async () => { + await fresh(); + const mp = path.join(dir, 'manifest.json'); + const m = JSON.parse(fs.readFileSync(mp, 'utf8')); + m.host_permissions.push('https://firestore.googleapis.com/*'); + m.version = '2.0'; + fs.writeFileSync(mp, JSON.stringify(m)); + const p = zipGateProblems(dir); + assert.ok(p['PERMISSION LOCK']); + assert.ok(p['VERSION GATE']); +}); + +test('a key in the bundle is refused', async () => { + await fresh(); + const swPath = path.join(dir, 'scripts/background/service-worker.js'); + const key = Buffer.from('AI' + 'zaSy' + 'Z'.repeat(33)).toString('base64'); + fs.appendFileSync(swPath, `\nconst _t = '${key}';\n`); + assert.ok(zipGateProblems(dir)['SECRET SCAN']); +}); + +test('gitState reports uncommitted changes', async () => { + const { execFileSync } = await import('node:child_process'); + const { gitState } = await import('../zip.mjs'); + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'proscan-git-')); + try { + const git = (...a) => execFileSync('git', a, { cwd: repo }); + git('init', '-q'); + fs.writeFileSync(path.join(repo, 'a.txt'), '1'); + git('add', '.'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'a'); + assert.deepEqual(gitState(repo).dirty, []); + fs.appendFileSync(path.join(repo, 'a.txt'), '2'); + assert.equal(gitState(repo).dirty.length, 1); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } +}); diff --git a/tools/version-gate.mjs b/tools/version-gate.mjs new file mode 100644 index 0000000..14476fa --- /dev/null +++ b/tools/version-gate.mjs @@ -0,0 +1,54 @@ +// tools/version-gate.mjs - the store rejects an upload whose version is not +// above the published one. Fails unless the manifest version is greater than +// tools/live-manifest.json. +// +// node tools/version-gate.mjs [manifest.json] + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { LIVE_MANIFEST } from './permission-lock.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +// Chrome versions are 1 to 4 dot-separated integers, 0 to 65535, no leading zeros. +export function parseVersion(v) { + if (typeof v !== 'string' || !/^(0|[1-9]\d{0,4})(\.(0|[1-9]\d{0,4})){0,3}$/.test(v)) { + throw new Error(`invalid Chrome version "${v}"`); + } + const parts = v.split('.').map(Number); + if (parts.some((n) => n > 65535)) throw new Error(`invalid Chrome version "${v}"`); + return parts; +} + +export function compareVersions(a, b) { + const pa = parseVersion(a); + const pb = parseVersion(b); + for (let i = 0; i < 4; i++) { + const d = (pa[i] ?? 0) - (pb[i] ?? 0); + if (d !== 0) return Math.sign(d); + } + return 0; +} + +function main(argv) { + const target = argv[0] ?? path.join(ROOT, 'manifest.json'); + const live = JSON.parse(fs.readFileSync(LIVE_MANIFEST, 'utf8')).version; + const built = JSON.parse(fs.readFileSync(target, 'utf8')).version; + let cmp; + try { + cmp = compareVersions(built, live); + } catch (err) { + console.error(`[version-gate] FAIL: ${err.message}`); + process.exit(1); + } + if (cmp <= 0) { + console.error(`[version-gate] FAIL: version ${built} is not above the live ${live}`); + process.exit(1); + } + console.log(`[version-gate] OK: ${built} > live ${live}`); +} + +const invokedDirectly = + process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) main(process.argv.slice(2)); diff --git a/tools/zip.mjs b/tools/zip.mjs index 3770c72..58bf163 100644 --- a/tools/zip.mjs +++ b/tools/zip.mjs @@ -1,51 +1,43 @@ -// tools/zip.mjs — packages dist/ ONLY into proscan-v{manifest.version}.zip -// at the repo root, behind three hard gates (each exits nonzero on failure): +// tools/zip.mjs - packages dist/ ONLY into +// dist-zips/proscan-v{version}-{sha}.zip, behind hard gates (any failure exits +// nonzero and writes nothing): // -// 1. ALLOWLIST — every file under dist/ must be in the explicit +// 1. ALLOWLIST - every file under dist/ must be in the explicit // allowlist (the exact build output set; no sourcemaps, // no strays). -// 2. LEGACY TRIPWIRE — no archive path may match a known legacy/dead-weight -// pattern (v1 root files, server/, tests, docs, etc.). -// 3. MANIFEST CLOSURE — every file manifest.json references must be present -// in the archive. +// 2. LEGACY TRIPWIRE - no archive path may match a known legacy/dead-weight +// pattern (v1 root files, server/, tests, docs, zips). +// 3. MANIFEST CLOSURE - every file manifest.json references must be present. +// 4. PROD ONLY - a dev build (emulator origins) never gets zipped. +// 5. PERMISSION LOCK - nothing added over tools/live-manifest.json. +// 6. VERSION GATE - version above the live one. +// 7. SECRET SCAN - no API keys in the archive. +// 8. CLEAN TREE - no uncommitted changes, so the zip matches a commit. +// `--allow-dirty` overrides it for local experiments and +// stamps the file name with -dirty. // -// Uses adm-zip; never shells out, never zips the repo root. +// Uses adm-zip; never shells out for packaging, never zips the repo root. import AdmZip from 'adm-zip'; import fs from 'node:fs'; import path from 'node:path'; +import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { COPY_FILES, BUNDLE_ENTRY, manifestClosureMissing, + htmlReferencesMissing, listFilesRecursive, } from './build.mjs'; +import { checkPermissionLock, LIVE_MANIFEST } from './permission-lock.mjs'; +import { compareVersions } from './version-gate.mjs'; +import { scanText } from './secret-scan.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const DIST = path.join(ROOT, 'dist'); +export const ZIP_DIR = path.join(ROOT, 'dist-zips'); -function fail(gate, lines) { - console.error(`[zip] FAIL (${gate}):`); - for (const line of lines) console.error(` - ${line}`); - process.exit(1); -} - -if (!fs.existsSync(path.join(DIST, 'manifest.json'))) { - fail('PRECONDITION', ['dist/ is missing or has no manifest.json — run `npm run build` first.']); -} - -// Archive contents = every file under dist/, recursively (forward-slash paths). -const archivePaths = listFilesRecursive(DIST).sort(); - -// ---- Gate 1: ALLOWLIST ----------------------------------------------------- -const ALLOWLIST = new Set([...COPY_FILES, BUNDLE_ENTRY]); -const notAllowed = archivePaths.filter((p) => !ALLOWLIST.has(p)); -if (notAllowed.length > 0) { - fail('ALLOWLIST', notAllowed.map((p) => `unexpected file in dist/: ${p}`)); -} - -// ---- Gate 2: LEGACY TRIPWIRE ----------------------------------------------- const LEGACY_PATTERNS = [ /^(popup|background|contentscript)\.(js|html|css)$/, // legacy v1 root files /^icon(16|48|128)\.png$/, // legacy root icons @@ -58,32 +50,105 @@ const LEGACY_PATTERNS = [ /(^|\/)node_modules(\/|$)/, /(^|\/)\.pytest_cache(\/|$)/, /(^|\/)docs\//, - /(^|\/)README[^/]*$/, - /(^|\/)CLAUDE[^/]*$/, + /\.md$/i, + /\.(zip|crx)$/i, + /\.map$/, + /(^|\/)\.git(\/|$)/, ]; -const legacyHits = archivePaths.filter((p) => LEGACY_PATTERNS.some((re) => re.test(p))); -if (legacyHits.length > 0) { - fail('LEGACY TRIPWIRE', legacyHits.map((p) => `legacy/dead-weight path in archive: ${p}`)); + +/** Returns {gate: [problems]} for a built dist dir; empty object means all pass. */ +export function zipGateProblems(distDir) { + const problems = {}; + const add = (gate, list) => { + if (list.length) problems[gate] = list; + }; + const archivePaths = listFilesRecursive(distDir).sort(); + + const allow = new Set([...COPY_FILES, BUNDLE_ENTRY, 'manifest.json']); + add('ALLOWLIST', archivePaths.filter((p) => !allow.has(p)).map((p) => `unexpected file: ${p}`)); + add('LEGACY TRIPWIRE', archivePaths + .filter((p) => LEGACY_PATTERNS.some((re) => re.test(p))) + .map((p) => `legacy/dead-weight path: ${p}`)); + add('MANIFEST CLOSURE', [...manifestClosureMissing(distDir), ...htmlReferencesMissing(distDir)]); + + const manifestText = fs.readFileSync(path.join(distDir, 'manifest.json'), 'utf8'); + const manifest = JSON.parse(manifestText); + add('PROD ONLY', /localhost|127\.0\.0\.1/.test(manifestText) + ? ['manifest references a local host; this is a dev build (rebuild without PROSCAN_ENV=dev)'] + : []); + + const live = JSON.parse(fs.readFileSync(LIVE_MANIFEST, 'utf8')); + add('PERMISSION LOCK', checkPermissionLock(manifest, live)); + let versionProblem = []; + try { + if (compareVersions(manifest.version, live.version) <= 0) { + versionProblem = [`version ${manifest.version} is not above the live ${live.version}`]; + } + } catch (err) { + versionProblem = [err.message]; + } + add('VERSION GATE', versionProblem); + + const secrets = []; + for (const rel of archivePaths.filter((p) => !p.endsWith('.png'))) { + for (const h of scanText(fs.readFileSync(path.join(distDir, rel), 'utf8'))) { + secrets.push(`${rel}:${h.line} ${h.name}`); + } + } + add('SECRET SCAN', secrets); + return problems; } -// ---- Gate 3: MANIFEST CLOSURE ---------------------------------------------- -// The archive contains exactly the files under dist/, so closure against -// dist/ is closure against the archive. -const missing = manifestClosureMissing(DIST); -if (missing.length > 0) { - fail('MANIFEST CLOSURE', missing.map((p) => `manifest references file not in archive: ${p}`)); +/** Returns {sha, dirty}; dirty lists `git status --porcelain` lines. */ +export function gitState(cwd = ROOT) { + const sha = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd }).toString().trim(); + const dirty = execFileSync('git', ['status', '--porcelain'], { cwd }).toString().split(/\r?\n/).filter(Boolean); + return { sha, dirty }; } -// ---- Package ---------------------------------------------------------------- -const manifest = JSON.parse(fs.readFileSync(path.join(DIST, 'manifest.json'), 'utf8')); -const outName = `proscan-v${manifest.version}.zip`; -const outPath = path.join(ROOT, outName); +function main(argv) { + const allowDirty = argv.includes('--allow-dirty'); + let git; + try { + git = gitState(); + } catch (err) { + console.error(`[zip] FAIL (CLEAN TREE): cannot read git state (${err.message}).`); + process.exit(1); + } + if (git.dirty.length && !allowDirty) { + console.error('[zip] FAIL (CLEAN TREE): uncommitted changes; commit them or pass --allow-dirty:'); + for (const line of git.dirty.slice(0, 20)) console.error(` - ${line}`); + process.exit(1); + } + + if (!fs.existsSync(path.join(DIST, 'manifest.json'))) { + console.error('[zip] FAIL: dist/ has no manifest.json. Run `npm run build` first.'); + process.exit(1); + } + const problems = zipGateProblems(DIST); + if (Object.keys(problems).length) { + for (const [gate, list] of Object.entries(problems)) { + console.error(`[zip] FAIL (${gate}):`); + for (const line of list) console.error(` - ${line}`); + } + process.exit(1); + } + + const archivePaths = listFilesRecursive(DIST).sort(); + const manifest = JSON.parse(fs.readFileSync(path.join(DIST, 'manifest.json'), 'utf8')); + const outName = `proscan-v${manifest.version}-${git.sha}${git.dirty.length ? '-dirty' : ''}.zip`; + fs.mkdirSync(ZIP_DIR, { recursive: true }); + + const zip = new AdmZip(); + for (const rel of archivePaths) { + const dir = path.posix.dirname(rel); + zip.addLocalFile(path.join(DIST, rel), dir === '.' ? '' : dir); + } + zip.writeZip(path.join(ZIP_DIR, outName)); -const zip = new AdmZip(); -for (const rel of archivePaths) { - const dir = path.posix.dirname(rel); - zip.addLocalFile(path.join(DIST, rel), dir === '.' ? '' : dir); + console.log(`[zip] OK: dist-zips/${outName} (${archivePaths.length} files, all gates passed).`); } -zip.writeZip(outPath); -console.log(`[zip] OK: ${outName} (${archivePaths.length} files, all gates passed).`); +const invokedDirectly = + process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (invokedDirectly) main(process.argv.slice(2));