From 26cf48ec90a0a1b8be10a1a75cafe78b0319739d Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 15:18:16 +0800 Subject: [PATCH 01/20] feat: add Google Safe Browsing support via Chrome real-time URL protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modern Chrome (v128+) uses the Safe Browsing v5 hash-real-time protocol to check URLs against Google's threat database on every navigation. This commit enables that by: - Installing Google Chrome in Docker (amd64 only, arm64 skipped) - Adding ensureSafeBrowsingPreferences() to seed profile with SB enabled - Modifying getPlaywrightLaunchOptions() to not suppress SB flags when GOOGLE_SAFE_BROWSING env var is set - Adding launchPersistentSafeContext() wrapper used by all crawler entry points - Logging SB status on first browser launch No local DB warmup or pre-seeding is needed — Chrome handles everything via real-time OHTTP lookups. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 57 ++++++++++++++++++++ Dockerfile | 44 +++++++++++++++- README.md | 31 +++++++++++ src/constants/common.ts | 70 ++++++++++++++++++++++--- src/crawlers/commonCrawlerFunc.ts | 4 +- src/crawlers/crawlIntelligentSitemap.ts | 4 +- src/crawlers/crawlLocalFile.ts | 3 +- src/crawlers/runCustom.ts | 3 +- src/npmIndex.ts | 3 +- 9 files changed, 205 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ec5ea56b..15dc19ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,8 +144,13 @@ The `constants` default export object holds runtime state: | `OOBEE_SCAN_PRODUCT` | Adds `scanProduct` tag to Sentry events | | `OOBEE_CONSECUTIVE_MAX_RETRIES` | Max consecutive HTTP failures before circuit breaker aborts crawl (default 100) | | `OOBEE_VALIDATE_URL` | If set, exit after URL validation without scanning | +<<<<<<< HEAD | `OOBEE_SAVE_DOM` | `1` or `true` = save full-page DOM HTML to `pageDOMs/` in results directory. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | | `OOBEE_SAVE_PAGE_SCREENSHOT` | `1` or `true` = save full-page desktop + mobile viewport screenshots to `pageDOMs/desktopPageScreenshots/` and `pageDOMs/mobilePageScreenshots/`. Mobile viewport uses iPhone 11 width programmatically. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | +======= +| `GOOGLE_SAFE_BROWSING` | `1` = enable Google Safe Browsing (requires Chrome, not Chromium) | +| `GOOGLE_SAFE_BROWSING_DEBUG` | `1` = enable verbose Safe Browsing debug logging | +>>>>>>> 86d9da36 (feat: add Google Safe Browsing support via Chrome real-time URL protection) | `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` | Proxy configuration | | `NO_PROXY` / `INCLUDE_PROXY` | Proxy bypass/include lists | @@ -180,6 +185,58 @@ The `constants` default export object holds runtime state: - Path separator differences in cookie profile regex - `CRAWLEE_SYSTEM_INFO_V2=1` needed (wmic deprecation) +## Safe Browsing + +### Overview + +Google Safe Browsing protects users by checking URLs against Google's threat database. Modern Chrome (v128+) uses the **v5 hash-real-time protocol** — it does NOT download a local threat database. Instead, URLs are checked in real-time via OHTTP (Oblivious HTTP) on every navigation. + +### Requirements + +1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys baked into the Chrome build. Chromium does not include them. +2. **`GOOGLE_SAFE_BROWSING=1`** environment variable — gates the feature. +3. **Network access** to `safebrowsing.googleapis.com` and `safebrowsingohttpgateway.googleapis.com`. + +### How It Works + +When `GOOGLE_SAFE_BROWSING=1` is set: + +1. `getPlaywrightLaunchOptions()` in `src/constants/common.ts` adds three Playwright default args to `ignoreDefaultArgs`: + - `--safebrowsing-disable-auto-update` (allows SB updater to run) + - `--disable-background-networking` (allows hash lookups) + - `--disable-client-side-phishing-detection` (allows phishing detection) + +2. `ensureSafeBrowsingPreferences()` writes `{ safebrowsing: { enabled: true, enhanced: false } }` into the Chrome profile's `Default/Preferences` before launch. Called from: + - `launchPersistentContextWithSafeBrowsing()` wrapper (used by all direct `launchPersistentContext` call sites) + - `getPreLaunchHook()` in `commonCrawlerFunc.ts` (Crawlee-managed browser pool) + +3. Chrome performs real-time hash-prefix lookups on every navigation. No warmup or pre-seeding is needed. + +### Docker / Architecture Constraints + +- **amd64 (x86_64)**: Google Chrome .deb is installed in Dockerfile. Safe Browsing works fully. +- **arm64 (aarch64)**: Google Chrome is NOT available for ARM64 Linux (as of July 2026). The Dockerfile skips Chrome installation. Safe Browsing is unavailable; only Chromium is present. +- Build the image with `docker build --platform linux/amd64` to ensure Chrome is available. + +### Environment Variables + +| Variable | Purpose | +|----------|---------| +| `GOOGLE_SAFE_BROWSING` | `1` = enable Safe Browsing (requires Chrome) | +| `GOOGLE_SAFE_BROWSING_DEBUG` | `1` = enable verbose Chrome Safe Browsing logging | + +### Key Files + +- `src/constants/common.ts` — `getPlaywrightLaunchOptions()`, `ensureSafeBrowsingPreferences()`, `launchPersistentContextWithSafeBrowsing()` +- `src/crawlers/commonCrawlerFunc.ts` — `getPreLaunchHook()` calls `ensureSafeBrowsingPreferences()` +- `Dockerfile` — Conditional Chrome installation for amd64 + +### What Does NOT Work + +- Chromium (Playwright's bundled browser) — lacks Safe Browsing entirely +- `--headless=old` or `--headless=new` for DB warmup — the old v4 local database approach is obsolete as of Chrome 128+ +- ARM64 Linux Docker — Chrome .deb not published for arm64 + ## Testing ```bash diff --git a/Dockerfile b/Dockerfile index 41d98695..cd7e9ec0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ zip && \ rm -rf /var/lib/apt/lists/* - + WORKDIR /app/oobee # Clone oobee repository @@ -31,6 +31,48 @@ RUN npm run build || true # true exits with code 0 - workaround for TS errors # Install Playwright browsers RUN npx playwright install chromium +# ============================================================================= +# Google Chrome installation for Safe Browsing support +# ============================================================================= +# WHY: Chrome's Safe Browsing (v5, hash-real-time protocol) protects users by +# checking URLs against Google's threat database in real-time via OHTTP. +# This is a Chrome-only feature — Chromium does NOT include it because it +# requires Google's proprietary API keys baked into the Chrome build. +# +# HOW IT WORKS (modern Chrome 128+): +# Chrome no longer downloads a local threat database (UrlSoceng.store.* files). +# Instead, it performs real-time hash-prefix lookups via the Safe Browsing v5 +# API using OHTTP (Oblivious HTTP) for privacy. This means: +# - No warmup/pre-seeding of a threat database is needed +# - Safe Browsing activates immediately on first navigation +# - The only requirements are: (1) Chrome (not Chromium), (2) safebrowsing +# enabled in Preferences, (3) network flags not suppressed +# +# ARCHITECTURE LIMITATION: +# Google Chrome .deb packages are only available for amd64 (x86_64). +# As of July 2026, Google has announced ARM64 Linux Chrome but has not yet +# published it to their apt repository or direct download URL. +# On arm64 builds, this step is skipped and Safe Browsing will not be available. +# +# TO ENABLE: Set env var GOOGLE_SAFE_BROWSING=1 when running the container. +# ============================================================================= +RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ + wget -q -O /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get update && apt-get install -y --no-install-recommends /tmp/chrome.deb && \ + rm -f /tmp/chrome.deb && rm -rf /var/lib/apt/lists/*; \ + else \ + echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ + fi + +# Pre-configure Safe Browsing preferences for Chrome profiles. +# When GOOGLE_SAFE_BROWSING=1, oobee's getPlaywrightLaunchOptions() will: +# 1. Stop ignoring --safebrowsing-disable-auto-update (lets SB updater run) +# 2. Stop ignoring --disable-background-networking (lets hash lookups work) +# 3. Stop ignoring --disable-client-side-phishing-detection +# The Preferences file below seeds any new Chrome profile with SB enabled. +RUN mkdir -p /tmp/oobee-sb-defaults/Default && \ + echo '{"safebrowsing":{"enabled":true,"enhanced":false}}' > /tmp/oobee-sb-defaults/Default/Preferences + # Add non-privileged user # Create a group named "purple" RUN groupadd -r purple diff --git a/README.md b/README.md index 4dec73c4..ac23e873 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ verapdf --version | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | | | NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | | | INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | | +| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing real-time URL protection. Requires Google Chrome (not Chromium). See [Safe Browsing](#safe-browsing) below. | | #### Environment variables used internally (Do not set) Do not set these environment variables or behaviour might change unexpectedly. @@ -581,6 +582,36 @@ For example, to conduct a website scan to the URL "http://localhost:8000" and wr npm run cli -- -c 2 -o oobee-scan-results.zip -u "http://localhost:8000" -w 360 ``` +## Safe Browsing + +Oobee can optionally enable Google Safe Browsing to protect against scanning malicious or phishing URLs. When enabled, Chrome checks every URL in real-time against Google's threat database before the page loads. + +### Requirements + +- **Google Chrome** (not Chromium) must be installed. Chromium does not include Safe Browsing. +- Set the `GOOGLE_SAFE_BROWSING` environment variable to any value. + +### Usage + +```bash +# macOS/Linux +GOOGLE_SAFE_BROWSING=1 node dist/cli.js -u https://example.com -b chrome + +# Docker (must be amd64 image — Chrome is not available for arm64 Linux) +docker run --platform linux/amd64 -e GOOGLE_SAFE_BROWSING=1 oobee node dist/cli.js -u https://example.com -b chrome +``` + +### How it works + +Modern Chrome (v128+) uses the Safe Browsing v5 protocol with real-time hash-prefix lookups via OHTTP. No local threat database is downloaded — URLs are checked on-the-fly during navigation. Oobee configures Chrome by: + +1. Enabling `safebrowsing` in the browser profile preferences +2. Removing Playwright's default flags that suppress Safe Browsing (`--safebrowsing-disable-auto-update`, `--disable-background-networking`, `--disable-client-side-phishing-detection`) + +### Docker architecture note + +Google Chrome `.deb` packages are only available for amd64 (x86_64). The Dockerfile conditionally installs Chrome on amd64 only. Build with `docker build --platform linux/amd64` to include Chrome and Safe Browsing support. + ## Report Once a scan of the site is completed. diff --git a/src/constants/common.ts b/src/constants/common.ts index 20c95aac..01ebbde2 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -404,7 +404,7 @@ const checkUrlConnectivityWithBrowser = async ( try { const launchPersistent = async () => { - browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, { + browserContext = await launchPersistentSafeContext(clonedDataDir, { ...launchOptions, ...contextOptions, }); @@ -929,7 +929,7 @@ const getRobotsTxtViaPlaywright = async ( try { if (process.env.CRAWLEE_HEADLESS === '1') { - browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, { + browserContext = await launchPersistentSafeContext(robotsDataDir, { ...getPlaywrightLaunchOptions(browser), ...(extraHTTPHeaders && { extraHTTPHeaders }), ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }), @@ -1151,7 +1151,7 @@ export const getLinksFromSitemap = async ( try { if (process.env.CRAWLEE_HEADLESS === '1') { - browserContext = await constants.launcher.launchPersistentContext( + browserContext = await launchPersistentSafeContext( finalUserDataDirectory, { ...getPlaywrightLaunchOptions(browser), @@ -2034,7 +2034,7 @@ export const submitFormViaPlaywright = async ( userDataDirectory: string, finalUrl: string, ) => { - const browserContext = await constants.launcher.launchPersistentContext(userDataDirectory, { + const browserContext = await launchPersistentSafeContext(userDataDirectory, { ...getPlaywrightLaunchOptions(browserToRun), }); @@ -2137,6 +2137,44 @@ export async function initModifiedUserAgent( const cacheProxyInfo = getProxyInfo(); +export function ensureSafeBrowsingPreferences(userDataDir: string): void { + if (!process.env.GOOGLE_SAFE_BROWSING || !userDataDir) return; + const defaultDir = path.join(userDataDir, 'Default'); + fs.mkdirSync(defaultDir, { recursive: true }); + const prefsPath = path.join(defaultDir, 'Preferences'); + let prefs: Record = {}; + if (fs.existsSync(prefsPath)) { + try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} + } + if (!(prefs.safebrowsing as Record)?.enabled) { + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: false }; + fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + } +} + +let safeBrowsingLoggedOnce = false; + +export async function launchPersistentSafeContext( + userDataDir: string, + options: Parameters[1], +) { + ensureSafeBrowsingPreferences(userDataDir); + + if (!safeBrowsingLoggedOnce && process.env.GOOGLE_SAFE_BROWSING) { + safeBrowsingLoggedOnce = true; + const chromeExists = fs.existsSync('/usr/bin/google-chrome') || + fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome') || + fs.existsSync('C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'); + if (chromeExists) { + consoleLogger.info('Google Safe Browsing enabled (real-time URL protection active)'); + } else { + consoleLogger.warn('GOOGLE_SAFE_BROWSING is set but Google Chrome was not found. Safe Browsing requires Chrome, not Chromium.'); + } + } + + return constants.launcher.launchPersistentContext(userDataDir, options); +} + /** * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser) * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch @@ -2187,10 +2225,18 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { break; } + const safeBrowsingEnabled = !!process.env.GOOGLE_SAFE_BROWSING; + + const baseIgnoredArgs = shouldIgnoreMuteAudio + ? ['--use-mock-keychain', '--mute-audio'] + : ['--use-mock-keychain']; + + const safeBrowsingIgnoredArgs = safeBrowsingEnabled + ? ['--safebrowsing-disable-auto-update', '--disable-client-side-phishing-detection', '--disable-background-networking'] + : []; + const options: LaunchOptions = { - ignoreDefaultArgs: shouldIgnoreMuteAudio - ? ['--use-mock-keychain', '--mute-audio'] - : ['--use-mock-keychain'], + ignoreDefaultArgs: [...baseIgnoredArgs, ...safeBrowsingIgnoredArgs], args: finalArgs, headless: process.env.CRAWLEE_HEADLESS === '1', ...(channel && { channel }), @@ -2203,6 +2249,16 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { consoleLogger.info(`Enabled browser slowMo with value: ${process.env.OOBEE_SLOWMO}ms`); } + if (safeBrowsingEnabled && !!process.env.GOOGLE_SAFE_BROWSING_DEBUG) { + options.args = [ + ...(options.args ?? []), + '--enable-logging=stderr', + '--log-level=0', + '--vmodule=safe_browsing*=2,*phishing*=2', + ]; + consoleLogger.info('Safe Browsing debug logging enabled'); + } + return options; }; diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index 7991e2e9..66582dfb 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -10,7 +10,7 @@ import { } from '../constants/constants.js'; import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js'; import { enrichColorContrastDOMContext, takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js'; -import { isFilePath } from '../constants/common.js'; +import { ensureSafeBrowsingPreferences, isFilePath } from '../constants/common.js'; import { extractAndGradeText } from './custom/extractAndGradeText.js'; import { ItemsInfo } from '../mergeAxeResults.js'; import { evaluateAltText } from './custom/evaluateAltText.js'; @@ -1300,6 +1300,8 @@ export const getPreLaunchHook = (userDataDirectory: string) => { // Silent fallback: use empty profile if clone fails } + ensureSafeBrowsingPreferences(effectiveDir); + // Clean any stale lock files that may block browser launches on Windows const lockFiles = [ path.join(effectiveDir, 'SingletonLock'), diff --git a/src/crawlers/crawlIntelligentSitemap.ts b/src/crawlers/crawlIntelligentSitemap.ts index 2fcccb77..ee1a25af 100644 --- a/src/crawlers/crawlIntelligentSitemap.ts +++ b/src/crawlers/crawlIntelligentSitemap.ts @@ -7,7 +7,7 @@ import { consoleLogger, guiInfoLog } from '../logs.js'; import crawlDomain from './crawlDomain.js'; import crawlSitemap from './crawlSitemap.js'; import { ViewportSettingsClass } from '../combine.js'; -import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent } from '../constants/common.js'; +import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent, launchPersistentSafeContext } from '../constants/common.js'; import { register } from '../utils.js'; const crawlIntelligentSitemap = async ( @@ -65,7 +65,7 @@ const crawlIntelligentSitemap = async ( if (process.env.CRAWLEE_HEADLESS === '1') { const effectiveUserDataDirectory = userDataDirectory || ''; - context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, { + context = await launchPersistentSafeContext(effectiveUserDataDirectory, { ...launchOptions, ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), ...(httpCredentials && { httpCredentials }), diff --git a/src/crawlers/crawlLocalFile.ts b/src/crawlers/crawlLocalFile.ts index 5e6d1742..83b11d6a 100644 --- a/src/crawlers/crawlLocalFile.ts +++ b/src/crawlers/crawlLocalFile.ts @@ -16,6 +16,7 @@ import { isFilePath, convertLocalFileToPath, convertPathToLocalFile, + launchPersistentSafeContext, } from '../constants/common.js'; import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js'; import { guiInfoLog } from '../logs.js'; @@ -151,7 +152,7 @@ export const crawlLocalFile = async ({ const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : ''; - const browserContext = await constants.launcher.launchPersistentContext( + const browserContext = await launchPersistentSafeContext( effectiveUserDataDirectory, { headless: process.env.CRAWLEE_HEADLESS === '1', diff --git a/src/crawlers/runCustom.ts b/src/crawlers/runCustom.ts index a3b8ae3c..c62c3e21 100644 --- a/src/crawlers/runCustom.ts +++ b/src/crawlers/runCustom.ts @@ -14,6 +14,7 @@ import { getBrowserToRun, getPlaywrightLaunchOptions, initModifiedUserAgent, + launchPersistentSafeContext, } from '../constants/common.js'; import { BrowserTypes } from '../constants/constants.js'; @@ -112,7 +113,7 @@ const runCustom = async ( const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders); - const context = await constants.launcher.launchPersistentContext(userDataDirectory, { + const context = await launchPersistentSafeContext(userDataDirectory, { ...baseLaunchOptions, args: mergedArgs, headless: false, diff --git a/src/npmIndex.ts b/src/npmIndex.ts index 7d1b252d..f72e061d 100644 --- a/src/npmIndex.ts +++ b/src/npmIndex.ts @@ -10,6 +10,7 @@ import { deleteClonedProfiles, getBrowserToRun, getPlaywrightLaunchOptions, + launchPersistentSafeContext, submitForm, } from './constants/common.js'; import { createCrawleeSubFolders, enrichViolationMessages, filterAxeResults } from './crawlers/commonCrawlerFunc.js'; @@ -433,7 +434,7 @@ export const init = async ({ browserToRun = browserData.browserToRun; clonedBrowserDataDir = browserData.clonedBrowserDataDir; - browserContext = await constants.launcher.launchPersistentContext(clonedBrowserDataDir, { + browserContext = await launchPersistentSafeContext(clonedBrowserDataDir, { viewport: viewportSettings, ...getPlaywrightLaunchOptions(browserToRun), }); From 0b56570e621f0a54264fb7fd2a7b0338849818b7 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 15:51:01 +0800 Subject: [PATCH 02/20] fix: restore Safe Browsing DB seeding for interstitial protection The real-time OHTTP approach alone doesn't trigger Chrome's interstitial warning pages in Playwright automation mode. The local hash-prefix DB (UrlSoceng.store.*, UrlMalware.store.*) is required as a first-pass filter. Restores safeBrowsingProfile.ts with improvements: - File lock to prevent concurrent warmup corruption - Process group kill for Chrome cleanup (SIGKILL -pid) - Xvfb lifecycle management on Linux (verify alive, kill after) - Windows: prints unsupported message and skips Also adds allowChromeErrors to urlGuard so Safe Browsing interstitial pages (chrome-error://) are not redirected away. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 43 +++--- README.md | 32 +---- src/constants/common.ts | 33 +---- src/crawlers/commonCrawlerFunc.ts | 4 +- src/crawlers/guards/urlGuard.ts | 8 +- src/crawlers/runCustom.ts | 2 +- src/safeBrowsingProfile.ts | 229 ++++++++++++++++++++++++++++++ 7 files changed, 264 insertions(+), 87 deletions(-) create mode 100644 src/safeBrowsingProfile.ts diff --git a/AGENTS.md b/AGENTS.md index 15dc19ce..041cf9d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -189,52 +189,57 @@ The `constants` default export object holds runtime state: ### Overview -Google Safe Browsing protects users by checking URLs against Google's threat database. Modern Chrome (v128+) uses the **v5 hash-real-time protocol** — it does NOT download a local threat database. Instead, URLs are checked in real-time via OHTTP (Oblivious HTTP) on every navigation. +Google Safe Browsing protects users by blocking navigation to phishing/malware URLs. It requires the local hash-prefix threat database (`UrlSoceng.store.*`, `UrlMalware.store.*`) to be present in the browser profile. Chrome uses this local DB as a first-pass filter and shows interstitial warning pages when a match is found. ### Requirements -1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys baked into the Chrome build. Chromium does not include them. -2. **`GOOGLE_SAFE_BROWSING=1`** environment variable — gates the feature. -3. **Network access** to `safebrowsing.googleapis.com` and `safebrowsingohttpgateway.googleapis.com`. +1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys. Chromium does not include them. +2. **`GOOGLE_SAFE_BROWSING`** environment variable set to any value. +3. **macOS or Linux only** — Windows is not yet supported (prints a warning and skips). ### How It Works -When `GOOGLE_SAFE_BROWSING=1` is set: +When `GOOGLE_SAFE_BROWSING` is set: -1. `getPlaywrightLaunchOptions()` in `src/constants/common.ts` adds three Playwright default args to `ignoreDefaultArgs`: - - `--safebrowsing-disable-auto-update` (allows SB updater to run) - - `--disable-background-networking` (allows hash lookups) - - `--disable-client-side-phishing-detection` (allows phishing detection) +1. `ensureAndInjectSafeBrowsing()` in `src/safeBrowsingProfile.ts` runs before each browser launch: + - **Fast path**: Copies the threat DB from your system Chrome profile (`~/Library/Application Support/Google/Chrome/Safe Browsing` on macOS, `~/.config/google-chrome/Safe Browsing` on Linux). This is instant. + - **Slow path**: If no system profile exists, spawns a real Chrome process for up to 120s to download the DB. On Linux without DISPLAY, tries Xvfb first, falls back to `--headless=old`. + - Writes `safebrowsing: { enabled: true }` into the profile's Preferences. + - Uses a file lock (`~/.oobee/safe-browsing-profile/.warmup-lock`) to prevent concurrent processes from corrupting the DB. -2. `ensureSafeBrowsingPreferences()` writes `{ safebrowsing: { enabled: true, enhanced: false } }` into the Chrome profile's `Default/Preferences` before launch. Called from: - - `launchPersistentContextWithSafeBrowsing()` wrapper (used by all direct `launchPersistentContext` call sites) - - `getPreLaunchHook()` in `commonCrawlerFunc.ts` (Crawlee-managed browser pool) +2. `getPlaywrightLaunchOptions()` in `src/constants/common.ts` adds three Playwright default args to `ignoreDefaultArgs`: + - `--safebrowsing-disable-auto-update` + - `--disable-background-networking` + - `--disable-client-side-phishing-detection` -3. Chrome performs real-time hash-prefix lookups on every navigation. No warmup or pre-seeding is needed. +3. `urlGuard.ts` allows `chrome-error://` protocol when Safe Browsing is active, so the interstitial warning page is not redirected away. ### Docker / Architecture Constraints - **amd64 (x86_64)**: Google Chrome .deb is installed in Dockerfile. Safe Browsing works fully. -- **arm64 (aarch64)**: Google Chrome is NOT available for ARM64 Linux (as of July 2026). The Dockerfile skips Chrome installation. Safe Browsing is unavailable; only Chromium is present. +- **arm64 (aarch64)**: Chrome is NOT available for ARM64 Linux. Safe Browsing is unavailable. - Build the image with `docker build --platform linux/amd64` to ensure Chrome is available. +- On first run in Docker, the slow path (spawning Chrome) is used since there's no system profile. ### Environment Variables | Variable | Purpose | |----------|---------| -| `GOOGLE_SAFE_BROWSING` | `1` = enable Safe Browsing (requires Chrome) | -| `GOOGLE_SAFE_BROWSING_DEBUG` | `1` = enable verbose Chrome Safe Browsing logging | +| `GOOGLE_SAFE_BROWSING` | Enable Safe Browsing (any value; requires Chrome) | +| `GOOGLE_SAFE_BROWSING_DEBUG` | Enable verbose Chrome Safe Browsing logging | ### Key Files -- `src/constants/common.ts` — `getPlaywrightLaunchOptions()`, `ensureSafeBrowsingPreferences()`, `launchPersistentContextWithSafeBrowsing()` -- `src/crawlers/commonCrawlerFunc.ts` — `getPreLaunchHook()` calls `ensureSafeBrowsingPreferences()` +- `src/safeBrowsingProfile.ts` — DB warmup, injection, and seeding logic +- `src/constants/common.ts` — `getPlaywrightLaunchOptions()` (ignoreDefaultArgs), `launchPersistentSafeContext()` wrapper +- `src/crawlers/guards/urlGuard.ts` — `allowChromeErrors` for interstitial pages +- `src/crawlers/runCustom.ts` — passes `allowChromeErrors` to urlGuard - `Dockerfile` — Conditional Chrome installation for amd64 ### What Does NOT Work - Chromium (Playwright's bundled browser) — lacks Safe Browsing entirely -- `--headless=old` or `--headless=new` for DB warmup — the old v4 local database approach is obsolete as of Chrome 128+ +- Windows — not yet supported (prints warning) - ARM64 Linux Docker — Chrome .deb not published for arm64 ## Testing diff --git a/README.md b/README.md index ac23e873..2effef6e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ verapdf --version | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | | | NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | | | INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | | -| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing real-time URL protection. Requires Google Chrome (not Chromium). See [Safe Browsing](#safe-browsing) below. | | +| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing URL protection. Requires Google Chrome (not Chromium) to be installed. Seeds the Safe Browsing threat database from your system Chrome profile on first run. macOS and Linux only (Windows not yet supported). | | #### Environment variables used internally (Do not set) Do not set these environment variables or behaviour might change unexpectedly. @@ -582,36 +582,6 @@ For example, to conduct a website scan to the URL "http://localhost:8000" and wr npm run cli -- -c 2 -o oobee-scan-results.zip -u "http://localhost:8000" -w 360 ``` -## Safe Browsing - -Oobee can optionally enable Google Safe Browsing to protect against scanning malicious or phishing URLs. When enabled, Chrome checks every URL in real-time against Google's threat database before the page loads. - -### Requirements - -- **Google Chrome** (not Chromium) must be installed. Chromium does not include Safe Browsing. -- Set the `GOOGLE_SAFE_BROWSING` environment variable to any value. - -### Usage - -```bash -# macOS/Linux -GOOGLE_SAFE_BROWSING=1 node dist/cli.js -u https://example.com -b chrome - -# Docker (must be amd64 image — Chrome is not available for arm64 Linux) -docker run --platform linux/amd64 -e GOOGLE_SAFE_BROWSING=1 oobee node dist/cli.js -u https://example.com -b chrome -``` - -### How it works - -Modern Chrome (v128+) uses the Safe Browsing v5 protocol with real-time hash-prefix lookups via OHTTP. No local threat database is downloaded — URLs are checked on-the-fly during navigation. Oobee configures Chrome by: - -1. Enabling `safebrowsing` in the browser profile preferences -2. Removing Playwright's default flags that suppress Safe Browsing (`--safebrowsing-disable-auto-update`, `--disable-background-networking`, `--disable-client-side-phishing-detection`) - -### Docker architecture note - -Google Chrome `.deb` packages are only available for amd64 (x86_64). The Dockerfile conditionally installs Chrome on amd64 only. Build with `docker build --platform linux/amd64` to include Chrome and Safe Browsing support. - ## Report Once a scan of the site is completed. diff --git a/src/constants/common.ts b/src/constants/common.ts index 01ebbde2..081afd0f 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -37,6 +37,7 @@ import { cleanUpAndExit, isFollowStrategy, randomThreeDigitNumberString, registe import { Answers, Data } from '../index.js'; import { DeviceDescriptor } from '../types/types.js'; import { getProxyInfo, proxyInfoToResolution, ProxySettings } from '../proxyService.js'; +import { ensureAndInjectSafeBrowsing } from '../safeBrowsingProfile.js'; // validateDirPath validates a provided directory path // returns null if no error @@ -2137,41 +2138,11 @@ export async function initModifiedUserAgent( const cacheProxyInfo = getProxyInfo(); -export function ensureSafeBrowsingPreferences(userDataDir: string): void { - if (!process.env.GOOGLE_SAFE_BROWSING || !userDataDir) return; - const defaultDir = path.join(userDataDir, 'Default'); - fs.mkdirSync(defaultDir, { recursive: true }); - const prefsPath = path.join(defaultDir, 'Preferences'); - let prefs: Record = {}; - if (fs.existsSync(prefsPath)) { - try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} - } - if (!(prefs.safebrowsing as Record)?.enabled) { - prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: false }; - fs.writeFileSync(prefsPath, JSON.stringify(prefs)); - } -} - -let safeBrowsingLoggedOnce = false; - export async function launchPersistentSafeContext( userDataDir: string, options: Parameters[1], ) { - ensureSafeBrowsingPreferences(userDataDir); - - if (!safeBrowsingLoggedOnce && process.env.GOOGLE_SAFE_BROWSING) { - safeBrowsingLoggedOnce = true; - const chromeExists = fs.existsSync('/usr/bin/google-chrome') || - fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome') || - fs.existsSync('C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'); - if (chromeExists) { - consoleLogger.info('Google Safe Browsing enabled (real-time URL protection active)'); - } else { - consoleLogger.warn('GOOGLE_SAFE_BROWSING is set but Google Chrome was not found. Safe Browsing requires Chrome, not Chromium.'); - } - } - + await ensureAndInjectSafeBrowsing(userDataDir); return constants.launcher.launchPersistentContext(userDataDir, options); } diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index 66582dfb..7991e2e9 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -10,7 +10,7 @@ import { } from '../constants/constants.js'; import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js'; import { enrichColorContrastDOMContext, takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js'; -import { ensureSafeBrowsingPreferences, isFilePath } from '../constants/common.js'; +import { isFilePath } from '../constants/common.js'; import { extractAndGradeText } from './custom/extractAndGradeText.js'; import { ItemsInfo } from '../mergeAxeResults.js'; import { evaluateAltText } from './custom/evaluateAltText.js'; @@ -1300,8 +1300,6 @@ export const getPreLaunchHook = (userDataDirectory: string) => { // Silent fallback: use empty profile if clone fails } - ensureSafeBrowsingPreferences(effectiveDir); - // Clean any stale lock files that may block browser launches on Windows const lockFiles = [ path.join(effectiveDir, 'SingletonLock'), diff --git a/src/crawlers/guards/urlGuard.ts b/src/crawlers/guards/urlGuard.ts index c4932a59..1c694489 100644 --- a/src/crawlers/guards/urlGuard.ts +++ b/src/crawlers/guards/urlGuard.ts @@ -1,7 +1,11 @@ const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']); export function addUrlGuardScript(context, opts = {}) { - const { fallbackUrl }: any = opts; + const { fallbackUrl, allowChromeErrors }: any = opts; + + const allowedProtocols = allowChromeErrors + ? new Set([...ALLOWED_PROTOCOLS, 'chrome-error:']) + : ALLOWED_PROTOCOLS; const lastAllowedUrlByPage = new WeakMap(); @@ -64,7 +68,7 @@ export function addUrlGuardScript(context, opts = {}) { return restoreToSafeUrl(page, urlStr); } - if (ALLOWED_PROTOCOLS.has(urlObj.protocol)) { + if (allowedProtocols.has(urlObj.protocol)) { lastAllowedUrlByPage.set(page, urlObj.toString()); return; } diff --git a/src/crawlers/runCustom.ts b/src/crawlers/runCustom.ts index c62c3e21..a2e03876 100644 --- a/src/crawlers/runCustom.ts +++ b/src/crawlers/runCustom.ts @@ -141,7 +141,7 @@ const runCustom = async ( // For handling closing playwright browser and continue generate artifacts etc registerSoftClose(processPageParams.stopAll); - addUrlGuardScript(context, { fallbackUrl: url }); + addUrlGuardScript(context, { fallbackUrl: url, allowChromeErrors: !!process.env.GOOGLE_SAFE_BROWSING }); const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage()); await initNewPage(page, pageClosePromises, processPageParams, pagesDict); diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts new file mode 100644 index 00000000..08892576 --- /dev/null +++ b/src/safeBrowsingProfile.ts @@ -0,0 +1,229 @@ +import { type ChildProcess, execSync, spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import printMessage from 'print-message'; +import { chromium as playwrightChromium } from 'playwright'; +import { consoleLogger } from './logs.js'; +import { messageOptions } from './constants/common.js'; + +const BASE_PROFILE_DIR = path.join(os.homedir(), '.oobee', 'safe-browsing-profile'); +const SB_DIR = path.join(BASE_PROFILE_DIR, 'Safe Browsing'); +const SEEDED_MARKER = '.sb-seeded'; +const LOCK_DIR = path.join(BASE_PROFILE_DIR, '.warmup-lock'); +const LOCK_STALE_MS = 180_000; + +function getChromeExecutable(): string { + const candidates: string[] = + process.platform === 'darwin' + ? ['/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'] + : [ + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + ]; + + const found = candidates.find(p => fs.existsSync(p)); + if (found) return found; + + try { + const playwrightPath = playwrightChromium.executablePath(); + if (fs.existsSync(playwrightPath)) return playwrightPath; + } catch {} + + return 'google-chrome'; +} + +function findSystemSafeBrowsingDir(): string | null { + const candidates = + process.platform === 'darwin' + ? [path.join(os.homedir(), 'Library', 'Application Support', 'Google', 'Chrome', 'Safe Browsing')] + : [ + path.join(os.homedir(), '.config', 'google-chrome', 'Safe Browsing'), + path.join(os.homedir(), '.config', 'chromium', 'Safe Browsing'), + ]; + return candidates.find(isDbDir) ?? null; +} + +function isDbDir(dir: string): boolean { + if (!fs.existsSync(dir)) return false; + return fs.readdirSync(dir).some(f => f.startsWith('UrlSoceng.store.') || f.startsWith('UrlMalware.store.')); +} + +function copyDirectory(src: string, dst: string): void { + fs.mkdirSync(dst, { recursive: true }); + for (const file of fs.readdirSync(src)) { + fs.copyFileSync(path.join(src, file), path.join(dst, file)); + } +} + +function killChromeTree(chrome: ChildProcess): void { + if (!chrome.pid) return; + try { + process.kill(-chrome.pid, 'SIGKILL'); + } catch { + try { chrome.kill('SIGKILL'); } catch {} + } +} + +function acquireLock(): boolean { + try { + fs.mkdirSync(LOCK_DIR); + fs.writeFileSync(path.join(LOCK_DIR, 'pid'), `${process.pid}\n${Date.now()}`); + return true; + } catch { + try { + const content = fs.readFileSync(path.join(LOCK_DIR, 'pid'), 'utf8'); + const timestamp = parseInt(content.split('\n')[1], 10); + if (Date.now() - timestamp > LOCK_STALE_MS) { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + return acquireLock(); + } + } catch { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + return acquireLock(); + } + return false; + } +} + +function releaseLock(): void { + try { fs.rmSync(LOCK_DIR, { recursive: true, force: true }); } catch {} +} + +async function spawnChromeForWarmup(): Promise { + printMessage(['Downloading Safe Browsing threat database via Chrome (up to 120s)...'], messageOptions); + + fs.mkdirSync(path.join(BASE_PROFILE_DIR, 'Default'), { recursive: true }); + fs.writeFileSync( + path.join(BASE_PROFILE_DIR, 'Default', 'Preferences'), + JSON.stringify({ safebrowsing: { enabled: true, enhanced: false } }), + ); + + const exe = getChromeExecutable(); + + const baseArgs = [ + `--user-data-dir=${BASE_PROFILE_DIR}`, + '--no-first-run', + '--no-default-browser-check', + '--disable-extensions', + ...(process.platform === 'linux' ? ['--no-sandbox', '--disable-setuid-sandbox'] : []), + ]; + + let spawnEnv: NodeJS.ProcessEnv = { ...process.env }; + let windowArgs: string[] = ['--window-position=-10000,-10000', '--window-size=1,1']; + let xvfbProcess: ChildProcess | null = null; + + if (process.platform === 'linux' && !process.env.DISPLAY) { + const displayNum = ':99'; + let xvfbStarted = false; + try { + xvfbProcess = spawn('Xvfb', [displayNum, '-screen', '0', '1024x768x24'], { + stdio: 'ignore', + detached: true, + }); + xvfbProcess.unref(); + await new Promise(r => setTimeout(r, 1500)); + + if (xvfbProcess.exitCode === null && !xvfbProcess.killed) { + spawnEnv = { ...process.env, DISPLAY: displayNum }; + xvfbStarted = true; + } else { + xvfbProcess = null; + } + } catch { + xvfbProcess = null; + } + + if (!xvfbStarted) { + windowArgs = ['--headless=old', '--disable-gpu']; + } + } + + const chrome = spawn( + exe, + [...baseArgs, ...windowArgs, 'about:blank'], + { stdio: 'ignore', detached: true, env: spawnEnv }, + ); + + const maxWait = 120_000; + const pollInterval = 5_000; + let waited = 0; + while (!isDbDir(SB_DIR) && waited < maxWait) { + await new Promise(r => setTimeout(r, pollInterval)); + waited += pollInterval; + } + + killChromeTree(chrome); + + if (xvfbProcess?.pid) { + try { process.kill(xvfbProcess.pid, 'SIGTERM'); } catch {} + } +} + +export async function warmupSafeBrowsingBaseProfile(): Promise { + if (isDbDir(SB_DIR)) return; + + fs.mkdirSync(BASE_PROFILE_DIR, { recursive: true }); + + const systemSbDir = findSystemSafeBrowsingDir(); + if (systemSbDir) { + printMessage(['Copying Safe Browsing threat database from system Chrome profile...'], messageOptions); + copyDirectory(systemSbDir, SB_DIR); + printMessage(['Google Safe Browsing enabled (real-time URL protection active)'], messageOptions); + return; + } + + if (!acquireLock()) { + consoleLogger.info('Another process is downloading Safe Browsing DB; waiting...'); + const waitStart = Date.now(); + while (!isDbDir(SB_DIR) && Date.now() - waitStart < 150_000) { + await new Promise(r => setTimeout(r, 5_000)); + } + return; + } + + try { + await spawnChromeForWarmup(); + + if (isDbDir(SB_DIR)) { + printMessage(['Google Safe Browsing enabled (real-time URL protection active)'], messageOptions); + } else { + printMessage(['WARNING: Safe Browsing DB did not populate. Protection may be reduced.'], messageOptions); + } + } finally { + releaseLock(); + } +} + +export function injectSafeBrowsingDb(targetDir: string): void { + if (!isDbDir(SB_DIR)) return; + if (fs.existsSync(path.join(targetDir, SEEDED_MARKER))) return; + + copyDirectory(SB_DIR, path.join(targetDir, 'Safe Browsing')); + + const defaultDir = path.join(targetDir, 'Default'); + fs.mkdirSync(defaultDir, { recursive: true }); + const prefsPath = path.join(defaultDir, 'Preferences'); + let prefs: Record = {}; + if (fs.existsSync(prefsPath)) { + try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} + } + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: false }; + fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + + fs.writeFileSync(path.join(targetDir, SEEDED_MARKER), new Date().toISOString()); +} + +export async function ensureAndInjectSafeBrowsing(targetDir: string): Promise { + if (!process.env.GOOGLE_SAFE_BROWSING) return; + + if (process.platform === 'win32') { + printMessage(['Google Safe Browsing is not yet supported on Windows.'], messageOptions); + return; + } + + await warmupSafeBrowsingBaseProfile(); + injectSafeBrowsingDb(targetDir); +} From 9fd20a8918e3510d329d2f4c0b7ff5a2fc4d75ef Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 16:43:11 +0800 Subject: [PATCH 03/20] fix: pre-seed Safe Browsing DB during Docker build with Xvfb Uses a proper bash script with exported DISPLAY to run Chrome + Xvfb during docker build. The DB downloads in ~145s and is baked into the image at /opt/oobee-safe-browsing/. At runtime, the DB is copied instantly from this location into browser profiles. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 52 ++++++++++++++++++++++---- pr-description.md | 76 ++++++++++++++++++++++++++++++++++++++ src/safeBrowsingProfile.ts | 1 + 3 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 pr-description.md diff --git a/Dockerfile b/Dockerfile index cd7e9ec0..7e21de1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,14 +64,50 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ fi -# Pre-configure Safe Browsing preferences for Chrome profiles. -# When GOOGLE_SAFE_BROWSING=1, oobee's getPlaywrightLaunchOptions() will: -# 1. Stop ignoring --safebrowsing-disable-auto-update (lets SB updater run) -# 2. Stop ignoring --disable-background-networking (lets hash lookups work) -# 3. Stop ignoring --disable-client-side-phishing-detection -# The Preferences file below seeds any new Chrome profile with SB enabled. -RUN mkdir -p /tmp/oobee-sb-defaults/Default && \ - echo '{"safebrowsing":{"enabled":true,"enhanced":false}}' > /tmp/oobee-sb-defaults/Default/Preferences +# Pre-seed Safe Browsing threat database into the image. +# Chrome needs to run non-headless to download the hash-prefix DB (UrlSoceng.store.*). +# We use Xvfb to provide a virtual display, then wait up to 180s for the DB to appear. +# The seeded DB at /opt/oobee-safe-browsing/ is copied into browser profiles at runtime. +COPY <<'SEEDSCRIPT' /tmp/seed-safe-browsing.sh +#!/bin/bash +set -e +apt-get update && apt-get install -y --no-install-recommends xvfb && rm -rf /var/lib/apt/lists/* +mkdir -p /opt/oobee-safe-browsing/Default +echo '{"safebrowsing":{"enabled":true,"enhanced":false}}' > /opt/oobee-safe-browsing/Default/Preferences +export DISPLAY=:99 +Xvfb :99 -screen 0 1024x768x24 & +XVFB_PID=$! +sleep 2 +google-chrome \ + --user-data-dir=/opt/oobee-safe-browsing \ + --no-first-run --no-default-browser-check --disable-extensions \ + --no-sandbox --disable-setuid-sandbox \ + --window-position=-10000,-10000 --window-size=1,1 \ + about:blank & +CHROME_PID=$! +echo "Waiting for Safe Browsing DB to download..." +WAITED=0 +while [ $WAITED -lt 180 ]; do + if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1 || \ + ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlMalware.store.* >/dev/null 2>&1; then + echo "Safe Browsing DB downloaded successfully (${WAITED}s)" + break + fi + sleep 5 + WAITED=$((WAITED + 5)) +done +kill $CHROME_PID 2>/dev/null || true +kill $XVFB_PID 2>/dev/null || true +if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1; then + echo "Safe Browsing DB baked into image" +else + echo "WARNING: Safe Browsing DB did not populate - will attempt at runtime" +fi +apt-get purge -y xvfb && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* +SEEDSCRIPT +RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome >/dev/null 2>&1; then \ + bash /tmp/seed-safe-browsing.sh; \ + fi && rm -f /tmp/seed-safe-browsing.sh # Add non-privileged user # Create a group named "purple" diff --git a/pr-description.md b/pr-description.md new file mode 100644 index 00000000..162ba6e7 --- /dev/null +++ b/pr-description.md @@ -0,0 +1,76 @@ +## feat: Page DOM and Screenshot Capture for UA Testing + +### Summary + +- Adds `OOBEE_SAVE_DOM` env var to save the full-page DOM HTML for each scanned page +- Adds `OOBEE_SAVE_PAGE_SCREENSHOT` env var to save full-page desktop and mobile viewport screenshots for each scanned page +- Generates a `dom-manifest.json` mapping each page URL to its hash, saved file paths, and any errors encountered during capture +- Supported across **all scan types**: Website, Sitemap, Intelligent, LocalFile, and Custom + +### How it works + +| Env Variable | Effect | +|---|---| +| `OOBEE_SAVE_DOM=1` | Saves full DOM to `/page-doms/-.html` | +| `OOBEE_SAVE_PAGE_SCREENSHOT=1` | Saves desktop PNG to `page-doms/desktop-page-screenshots/` and mobile PNG to `page-doms/mobile-page-screenshots/` | + +- **Hash**: 7-character SHA-256 of the page URL (consistent, like git short hashes) +- **Truncated path**: URL pathname with `/` replaced by `_`, max 80 chars, for human readability +- **Filename collisions**: Resolved by appending `-2`, `-3`, etc. before the file extension +- **Mobile viewport**: Width derived programmatically from Playwright's `devices['iPhone 11']` profile. The viewport is resized in-place (no separate tab), then restored to the original desktop size +- **Manifest**: `page-doms/dom-manifest.json` is written when either env var is enabled + +### Output structure + +``` +results//page-doms/ +├── -.html # Full DOM (OOBEE_SAVE_DOM) +├── dom-manifest.json # URL → file path mapping +├── desktop-page-screenshots/ +│ └── -.png +└── mobile-page-screenshots/ + └── -.png +``` + +### Example `dom-manifest.json` + +```json +{ + "generatedAt": "2026-07-06T13:22:06.000Z", + "pages": [ + { + "url": "https://example.com/about", + "hash": "a1b2c3d", + "domFile": "page-doms/a1b2c3d-about.html", + "desktopScreenshot": "page-doms/desktop-page-screenshots/a1b2c3d-about.png", + "mobileScreenshot": "page-doms/mobile-page-screenshots/a1b2c3d-about.png", + "errors": [] + } + ] +} +``` + +### Files changed + +| File | Change | +|------|--------| +| `src/crawlers/pageCapture.ts` | **New** — core module for DOM/screenshot capture and manifest generation | +| `src/crawlers/crawlDomain.ts` | Calls `capturePageData()` after axe scan | +| `src/crawlers/crawlSitemap.ts` | Calls `capturePageData()` after axe scan | +| `src/crawlers/crawlLocalFile.ts` | Calls `capturePageData()` after axe scan | +| `src/crawlers/custom/utils.ts` | Calls `capturePageData()` after axe scan | +| `src/combine.ts` | Calls `writeManifest()` + `resetCaptureEntries()` post-crawl | +| `README.md` | Documents new env vars | +| `AGENTS.md` | Documents new env vars and output structure | + +### Test plan + +- [ ] Run sitemap scan with `OOBEE_SAVE_DOM=1` — verify `.html` files appear in `page-doms/` +- [ ] Run website scan with `OOBEE_SAVE_PAGE_SCREENSHOT=1` — verify desktop and mobile PNGs +- [ ] Run with both env vars set — verify `dom-manifest.json` contains all entries +- [ ] Verify filenames are flat (no nested subdirectories under `page-doms/`) +- [ ] Verify mobile screenshots use iPhone 11 width (375px) +- [ ] Verify desktop viewport is restored after mobile screenshot +- [ ] Run without env vars set — verify no `page-doms/` directory is created +- [ ] Test URL with long path (>80 chars) — verify truncation +- [ ] Test filename collision scenario — verify `-2` suffix applied diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts index 08892576..e7cb629b 100644 --- a/src/safeBrowsingProfile.ts +++ b/src/safeBrowsingProfile.ts @@ -40,6 +40,7 @@ function findSystemSafeBrowsingDir(): string | null { process.platform === 'darwin' ? [path.join(os.homedir(), 'Library', 'Application Support', 'Google', 'Chrome', 'Safe Browsing')] : [ + '/opt/oobee-safe-browsing/Safe Browsing', path.join(os.homedir(), '.config', 'google-chrome', 'Safe Browsing'), path.join(os.homedir(), '.config', 'chromium', 'Safe Browsing'), ]; From e715dcd83fd1a6f2b8128121f3d85d7795c73e69 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 16:44:24 +0800 Subject: [PATCH 04/20] chore: remove stray pr-description.md Co-Authored-By: Claude Opus 4.6 (1M context) --- pr-description.md | 76 ----------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 pr-description.md diff --git a/pr-description.md b/pr-description.md deleted file mode 100644 index 162ba6e7..00000000 --- a/pr-description.md +++ /dev/null @@ -1,76 +0,0 @@ -## feat: Page DOM and Screenshot Capture for UA Testing - -### Summary - -- Adds `OOBEE_SAVE_DOM` env var to save the full-page DOM HTML for each scanned page -- Adds `OOBEE_SAVE_PAGE_SCREENSHOT` env var to save full-page desktop and mobile viewport screenshots for each scanned page -- Generates a `dom-manifest.json` mapping each page URL to its hash, saved file paths, and any errors encountered during capture -- Supported across **all scan types**: Website, Sitemap, Intelligent, LocalFile, and Custom - -### How it works - -| Env Variable | Effect | -|---|---| -| `OOBEE_SAVE_DOM=1` | Saves full DOM to `/page-doms/-.html` | -| `OOBEE_SAVE_PAGE_SCREENSHOT=1` | Saves desktop PNG to `page-doms/desktop-page-screenshots/` and mobile PNG to `page-doms/mobile-page-screenshots/` | - -- **Hash**: 7-character SHA-256 of the page URL (consistent, like git short hashes) -- **Truncated path**: URL pathname with `/` replaced by `_`, max 80 chars, for human readability -- **Filename collisions**: Resolved by appending `-2`, `-3`, etc. before the file extension -- **Mobile viewport**: Width derived programmatically from Playwright's `devices['iPhone 11']` profile. The viewport is resized in-place (no separate tab), then restored to the original desktop size -- **Manifest**: `page-doms/dom-manifest.json` is written when either env var is enabled - -### Output structure - -``` -results//page-doms/ -├── -.html # Full DOM (OOBEE_SAVE_DOM) -├── dom-manifest.json # URL → file path mapping -├── desktop-page-screenshots/ -│ └── -.png -└── mobile-page-screenshots/ - └── -.png -``` - -### Example `dom-manifest.json` - -```json -{ - "generatedAt": "2026-07-06T13:22:06.000Z", - "pages": [ - { - "url": "https://example.com/about", - "hash": "a1b2c3d", - "domFile": "page-doms/a1b2c3d-about.html", - "desktopScreenshot": "page-doms/desktop-page-screenshots/a1b2c3d-about.png", - "mobileScreenshot": "page-doms/mobile-page-screenshots/a1b2c3d-about.png", - "errors": [] - } - ] -} -``` - -### Files changed - -| File | Change | -|------|--------| -| `src/crawlers/pageCapture.ts` | **New** — core module for DOM/screenshot capture and manifest generation | -| `src/crawlers/crawlDomain.ts` | Calls `capturePageData()` after axe scan | -| `src/crawlers/crawlSitemap.ts` | Calls `capturePageData()` after axe scan | -| `src/crawlers/crawlLocalFile.ts` | Calls `capturePageData()` after axe scan | -| `src/crawlers/custom/utils.ts` | Calls `capturePageData()` after axe scan | -| `src/combine.ts` | Calls `writeManifest()` + `resetCaptureEntries()` post-crawl | -| `README.md` | Documents new env vars | -| `AGENTS.md` | Documents new env vars and output structure | - -### Test plan - -- [ ] Run sitemap scan with `OOBEE_SAVE_DOM=1` — verify `.html` files appear in `page-doms/` -- [ ] Run website scan with `OOBEE_SAVE_PAGE_SCREENSHOT=1` — verify desktop and mobile PNGs -- [ ] Run with both env vars set — verify `dom-manifest.json` contains all entries -- [ ] Verify filenames are flat (no nested subdirectories under `page-doms/`) -- [ ] Verify mobile screenshots use iPhone 11 width (375px) -- [ ] Verify desktop viewport is restored after mobile screenshot -- [ ] Run without env vars set — verify no `page-doms/` directory is created -- [ ] Test URL with long path (>80 chars) — verify truncation -- [ ] Test filename collision scenario — verify `-2` suffix applied From f8ef8ab5e1f4042625102220d0350223b0b27e93 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 16:59:35 +0800 Subject: [PATCH 05/20] fix: early exit with warning when Chrome not found for Safe Browsing Avoids a wasted 120s warmup timeout on arm64 Linux or environments without Chrome installed. Prints a clear message directing users to build with --platform linux/amd64. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/safeBrowsingProfile.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts index e7cb629b..fec4b57c 100644 --- a/src/safeBrowsingProfile.ts +++ b/src/safeBrowsingProfile.ts @@ -176,6 +176,13 @@ export async function warmupSafeBrowsingBaseProfile(): Promise { return; } + const exe = getChromeExecutable(); + const chromeFound = fs.existsSync(exe); + if (!chromeFound) { + printMessage(['WARNING: Google Chrome not found. Safe Browsing requires Chrome (not Chromium). On Linux Docker, build with --platform linux/amd64.'], messageOptions); + return; + } + if (!acquireLock()) { consoleLogger.info('Another process is downloading Safe Browsing DB; waiting...'); const waitStart = Date.now(); From 95353cc6a16a7cd310013ab27102fdf7191aa590 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 17:16:12 +0800 Subject: [PATCH 06/20] fix: add Linux Chrome data dir detection in getBrowserToRun getDefaultChromeDataDir() only checked Windows/macOS. On Linux with Chrome installed, it returned null causing fallback to Chromium. Now detects /usr/bin/google-chrome and returns ~/.config/google-chrome. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/constants/constants.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 387d1be8..5568ca7b 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -86,6 +86,17 @@ export const getDefaultChromeDataDir = (): string => { if (defaultChromeDataDir && fs.existsSync(defaultChromeDataDir)) { return defaultChromeDataDir; } + + // Linux: check if Chrome is installed (data dir may not exist yet — create it) + if (os.platform() === 'linux') { + const chromeExists = fs.existsSync('/usr/bin/google-chrome') || fs.existsSync('/usr/bin/google-chrome-stable'); + if (chromeExists) { + const linuxChromeDataDir = path.join(os.homedir(), '.config', 'google-chrome'); + fs.mkdirSync(linuxChromeDataDir, { recursive: true }); + return linuxChromeDataDir; + } + } + return null; } catch (error) { console.error(`Error in getDefaultChromeDataDir(): ${error}`); From ab63f8b41902b0023c8524a7a9d669837d5649ee Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 17:17:55 +0800 Subject: [PATCH 07/20] fix: Linux Chrome data dir uses same path as Chromium (./Chromium Support) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/constants/constants.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 5568ca7b..49415f10 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -87,12 +87,16 @@ export const getDefaultChromeDataDir = (): string => { return defaultChromeDataDir; } - // Linux: check if Chrome is installed (data dir may not exist yet — create it) + // Linux: check if Chrome is installed; use same scratch dir pattern as Chromium if (os.platform() === 'linux') { const chromeExists = fs.existsSync('/usr/bin/google-chrome') || fs.existsSync('/usr/bin/google-chrome-stable'); if (chromeExists) { - const linuxChromeDataDir = path.join(os.homedir(), '.config', 'google-chrome'); - fs.mkdirSync(linuxChromeDataDir, { recursive: true }); + let linuxChromeDataDir = path.join(process.cwd(), 'Chromium Support'); + try { + fs.mkdirSync(linuxChromeDataDir, { recursive: true }); + } catch { + linuxChromeDataDir = '/tmp'; + } return linuxChromeDataDir; } } From 4d512c9ea6a4fbbcf29ba820090768ef10c3b67c Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 17:54:41 +0800 Subject: [PATCH 08/20] fix: create minimal Local State file for Linux Chrome profile cloneChromeProfiles fails when no Local State file exists in the data dir, which is always the case in fresh Docker containers. Creating a minimal one prevents the warning and allows the scan to proceed. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/constants/constants.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 49415f10..be7c8da9 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -97,6 +97,11 @@ export const getDefaultChromeDataDir = (): string => { } catch { linuxChromeDataDir = '/tmp'; } + // Create minimal Local State file so cloneChromeProfiles succeeds + const localStatePath = path.join(linuxChromeDataDir, 'Local State'); + if (!fs.existsSync(localStatePath)) { + fs.writeFileSync(localStatePath, JSON.stringify({ profile: { info_cache: {} } })); + } return linuxChromeDataDir; } } From dc85c2655952104ea147bbc13121f9880ed1081e Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 17:55:59 +0800 Subject: [PATCH 09/20] fix: prevent crash in cloneChromeProfileCookieFiles on Linux The function only handles win32/darwin cookie paths. On Linux, profileCookiesDir was undefined causing a TypeError on .length. Now returns true early on Linux (no cookies to clone). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/constants/common.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/constants/common.ts b/src/constants/common.ts index 081afd0f..7b7b90fe 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -1593,6 +1593,8 @@ const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, d ignore: 'oobee*/**', }); profileNamesRegex = /Chrome\/(.*?)\/Cookies/; + } else { + return true; } if (profileCookiesDir.length > 0) { From 689b1250938591dfc3ff552282893a044e9611a4 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 21:49:06 +0800 Subject: [PATCH 10/20] fix: enable safe browsing for crawlDomain and crawlSitemap browser pools getPreLaunchHook now injects the Safe Browsing threat database into each browser pool instance's user data directory. Both crawlDomain and crawlSitemap handle chrome-error:// interstitials (blocked pages) by logging them as excluded with STATUS_CODE_METADATA[3]. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/constants/constants.ts | 1 + src/crawlers/commonCrawlerFunc.ts | 5 +++++ src/crawlers/crawlDomain.ts | 16 ++++++++++++++++ src/crawlers/crawlSitemap.ts | 15 +++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/src/constants/constants.ts b/src/constants/constants.ts index be7c8da9..17c0723b 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -1020,6 +1020,7 @@ export const STATUS_CODE_METADATA: Record = { 0: 'Page Excluded', 1: 'Not A Supported Document', 2: 'Web Crawler Errored', + 3: 'Blocked by Safe Browsing', // 599 is set because Crawlee returns response status 100, 102, 103 as 599 599: 'Uncommon Response Status Code Received', diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index 7991e2e9..b6623895 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -23,6 +23,7 @@ import xPathToCss from './custom/xPathToCss.js'; import type { Response as PlaywrightResponse } from 'playwright'; import fs from 'fs'; import { getStoragePath } from '../utils.js'; +import { injectSafeBrowsingDb } from '../safeBrowsingProfile.js'; import path from 'path'; // types @@ -1247,6 +1248,10 @@ export const getPreLaunchHook = (userDataDirectory: string) => { await fsp.mkdir(effectiveDir, { recursive: true }); + if (process.env.GOOGLE_SAFE_BROWSING) { + injectSafeBrowsingDb(effectiveDir); + } + // Copy auth-relevant files from the pristine base directory so // authenticated sessions are preserved across pool rotations. try { diff --git a/src/crawlers/crawlDomain.ts b/src/crawlers/crawlDomain.ts index 2c1f3b19..80bf41d6 100644 --- a/src/crawlers/crawlDomain.ts +++ b/src/crawlers/crawlDomain.ts @@ -522,6 +522,22 @@ const crawlDomain = async ({ actualUrl = page.url(); } + if (actualUrl.startsWith('chrome-error:')) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + await enqueueProcess(page, enqueueLinks, browserContext); + return; + } + // Second-pass requests: only do click-discovery, skip scanning if (request.label?.startsWith('__clickpass__')) { await enqueueProcess(page, enqueueLinks, browserContext); diff --git a/src/crawlers/crawlSitemap.ts b/src/crawlers/crawlSitemap.ts index 8e2fc368..3c6916d7 100644 --- a/src/crawlers/crawlSitemap.ts +++ b/src/crawlers/crawlSitemap.ts @@ -275,6 +275,21 @@ const crawlSitemap = async ({ const actualUrl = page.url() || request.loadedUrl || request.url; + if (actualUrl.startsWith('chrome-error:')) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000; From 3e0228cc57b73a142f9acd98e2b091fb77ecb254 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 21:51:41 +0800 Subject: [PATCH 11/20] fix: grant read permissions on pre-seeded Safe Browsing DB for non-root user The Safe Browsing DB at /opt/oobee-safe-browsing/ is seeded as root during docker build, but the app runs as the unprivileged 'purple' user. Without world-readable permissions, Node's fs.readdirSync fails with EACCES and Safe Browsing protection is silently disabled. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7e21de1d..fac5140d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,7 +107,8 @@ apt-get purge -y xvfb && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* SEEDSCRIPT RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome >/dev/null 2>&1; then \ bash /tmp/seed-safe-browsing.sh; \ - fi && rm -f /tmp/seed-safe-browsing.sh + fi && rm -f /tmp/seed-safe-browsing.sh && \ + chmod -R a+rX /opt/oobee-safe-browsing 2>/dev/null || true # Add non-privileged user # Create a group named "purple" From b4559d11098489b2a2e8770bb465c7fcea057f51 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 22:24:10 +0800 Subject: [PATCH 12/20] fix: allow Chrome Safe Browsing service to initialize before URL check On fresh Docker containers, Chrome's real-time Safe Browsing v5 (OHTTP) needs time to connect to Google's relay servers after launch. Add a 10s warm-up delay before navigating to the target URL when GOOGLE_SAFE_BROWSING is enabled. Also switch to Enhanced Safe Browsing (enhanced: true) for more aggressive real-time URL checking. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 2 +- src/constants/common.ts | 8 ++++++++ src/safeBrowsingProfile.ts | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index fac5140d..c8577e8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ COPY <<'SEEDSCRIPT' /tmp/seed-safe-browsing.sh set -e apt-get update && apt-get install -y --no-install-recommends xvfb && rm -rf /var/lib/apt/lists/* mkdir -p /opt/oobee-safe-browsing/Default -echo '{"safebrowsing":{"enabled":true,"enhanced":false}}' > /opt/oobee-safe-browsing/Default/Preferences +echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > /opt/oobee-safe-browsing/Default/Preferences export DISPLAY=:99 Xvfb :99 -screen 0 1024x768x24 & XVFB_PID=$! diff --git a/src/constants/common.ts b/src/constants/common.ts index 7b7b90fe..3c4664e8 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -467,6 +467,14 @@ const checkUrlConnectivityWithBrowser = async ( consoleLogger.info(`Unable to set download deny: ${(e as Error).message}`); } + // Give Chrome's Safe Browsing real-time service time to initialize. + // On fresh Docker containers, Chrome needs a few seconds to connect to + // Google's OHTTP relay and establish the v5 hash-prefix lookup service. + if (process.env.GOOGLE_SAFE_BROWSING) { + await page.goto('about:blank'); + await new Promise(resolve => setTimeout(resolve, 10000)); + } + // STEP 2: Navigate (follows server-side redirects) page.once('download', () => { res.status = constants.urlCheckStatuses.notASupportedDocument.code; diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts index fec4b57c..f589c97e 100644 --- a/src/safeBrowsingProfile.ts +++ b/src/safeBrowsingProfile.ts @@ -99,7 +99,7 @@ async function spawnChromeForWarmup(): Promise { fs.mkdirSync(path.join(BASE_PROFILE_DIR, 'Default'), { recursive: true }); fs.writeFileSync( path.join(BASE_PROFILE_DIR, 'Default', 'Preferences'), - JSON.stringify({ safebrowsing: { enabled: true, enhanced: false } }), + JSON.stringify({ safebrowsing: { enabled: true, enhanced: true } }), ); const exe = getChromeExecutable(); @@ -218,7 +218,7 @@ export function injectSafeBrowsingDb(targetDir: string): void { if (fs.existsSync(prefsPath)) { try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} } - prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: false }; + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: true }; fs.writeFileSync(prefsPath, JSON.stringify(prefs)); fs.writeFileSync(path.join(targetDir, SEEDED_MARKER), new Date().toISOString()); From dba44133aafb3e8373828fe45ee90fb89b523a47 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Thu, 9 Jul 2026 23:56:06 +0800 Subject: [PATCH 13/20] fix: use Xvfb headful mode for Safe Browsing in Docker Chrome's Safe Browsing does not enforce interstitial blocking in headless mode. When GOOGLE_SAFE_BROWSING is enabled on Linux without a display, start Xvfb and force headful mode so Chrome renders to a virtual framebuffer and enforces real-time URL protection. Also removes the legacy DB seed script from Dockerfile (Chrome 128+ uses real-time v5 API, not local UrlSoceng.store files) and adds debug logging throughout the Safe Browsing flow. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 88 ++++++++------------------------------ src/constants/common.ts | 63 ++++++++++++++++++++++++--- src/safeBrowsingProfile.ts | 47 ++++++++++++++++++-- 3 files changed, 118 insertions(+), 80 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8577e8f..700e2e0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,28 +6,10 @@ FROM mcr.microsoft.com/playwright:v1.61.1-noble RUN apt-get update && apt-get install -y --no-install-recommends \ git \ unzip \ - zip && \ + zip \ + xvfb && \ rm -rf /var/lib/apt/lists/* -WORKDIR /app/oobee - -# Clone oobee repository -# RUN git clone --branch master https://github.com/GovTechSG/oobee.git /app/oobee - -# OR Copy oobee files from local directory -COPY . . - -# Environment variables for node and Playwright -ENV NODE_ENV=production -ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="true" - -# Install oobee dependencies -# TODO: Move back to npm ci --omit=dev once module package-lock issue vs MacOS is resolved -RUN npm install --omit=dev - -# Compile TypeScript for oobee -RUN npm run build || true # true exits with code 0 - workaround for TS errors - # Install Playwright browsers RUN npx playwright install chromium @@ -64,51 +46,21 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ fi -# Pre-seed Safe Browsing threat database into the image. -# Chrome needs to run non-headless to download the hash-prefix DB (UrlSoceng.store.*). -# We use Xvfb to provide a virtual display, then wait up to 180s for the DB to appear. -# The seeded DB at /opt/oobee-safe-browsing/ is copied into browser profiles at runtime. -COPY <<'SEEDSCRIPT' /tmp/seed-safe-browsing.sh -#!/bin/bash -set -e -apt-get update && apt-get install -y --no-install-recommends xvfb && rm -rf /var/lib/apt/lists/* -mkdir -p /opt/oobee-safe-browsing/Default -echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > /opt/oobee-safe-browsing/Default/Preferences -export DISPLAY=:99 -Xvfb :99 -screen 0 1024x768x24 & -XVFB_PID=$! -sleep 2 -google-chrome \ - --user-data-dir=/opt/oobee-safe-browsing \ - --no-first-run --no-default-browser-check --disable-extensions \ - --no-sandbox --disable-setuid-sandbox \ - --window-position=-10000,-10000 --window-size=1,1 \ - about:blank & -CHROME_PID=$! -echo "Waiting for Safe Browsing DB to download..." -WAITED=0 -while [ $WAITED -lt 180 ]; do - if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1 || \ - ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlMalware.store.* >/dev/null 2>&1; then - echo "Safe Browsing DB downloaded successfully (${WAITED}s)" - break - fi - sleep 5 - WAITED=$((WAITED + 5)) -done -kill $CHROME_PID 2>/dev/null || true -kill $XVFB_PID 2>/dev/null || true -if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1; then - echo "Safe Browsing DB baked into image" -else - echo "WARNING: Safe Browsing DB did not populate - will attempt at runtime" -fi -apt-get purge -y xvfb && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* -SEEDSCRIPT -RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome >/dev/null 2>&1; then \ - bash /tmp/seed-safe-browsing.sh; \ - fi && rm -f /tmp/seed-safe-browsing.sh && \ - chmod -R a+rX /opt/oobee-safe-browsing 2>/dev/null || true +# --- App code (changes here don't invalidate Chrome layer above) --- + +WORKDIR /app/oobee + +# Environment variables for node and Playwright +ENV NODE_ENV=production +ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="true" + +# Install dependencies first (cached unless package.json/package-lock.json change) +COPY package.json package-lock.json ./ +RUN npm install --omit=dev + +# Copy source and compile TypeScript +COPY . . +RUN npm run build || true # true exits with code 0 - workaround for TS errors # Add non-privileged user # Create a group named "purple" @@ -125,12 +77,6 @@ WORKDIR /app # Set the ownership of the oobee directory to the user "purple" RUN chown -R purple:purple /app -# Copy any application and support files -# COPY . . - -# Install any app dependencies for your application -# RUN npm ci --omit=dev - # For oobee to be run from present working directory, comment out as necessary WORKDIR /app/oobee diff --git a/src/constants/common.ts b/src/constants/common.ts index 3c4664e8..b0701bc8 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -14,6 +14,7 @@ import url, { fileURLToPath, pathToFileURL } from 'url'; import safe from 'safe-regex'; import * as https from 'https'; import os from 'os'; +import { execSync as execSyncFn } from 'child_process'; import mime from 'mime'; import { minimatch } from 'minimatch'; import { globSync, GlobOptionsWithFileTypesFalse } from 'glob'; @@ -471,8 +472,10 @@ const checkUrlConnectivityWithBrowser = async ( // On fresh Docker containers, Chrome needs a few seconds to connect to // Google's OHTTP relay and establish the v5 hash-prefix lookup service. if (process.env.GOOGLE_SAFE_BROWSING) { + consoleLogger.info('[SafeBrowsing] Warming up Chrome Safe Browsing service (10s)...'); await page.goto('about:blank'); await new Promise(resolve => setTimeout(resolve, 10000)); + consoleLogger.info('[SafeBrowsing] Warm-up complete, navigating to target URL...'); } // STEP 2: Navigate (follows server-side redirects) @@ -480,15 +483,38 @@ const checkUrlConnectivityWithBrowser = async ( res.status = constants.urlCheckStatuses.notASupportedDocument.code; return res; }); - + // OPTIMIZATION: Wait for 'domcontentloaded' only - const response = await page.goto(url, { - timeout: 15000, - waitUntil: 'domcontentloaded', // enough to get status + allow potential client redirects to kick in - }); + let response; + try { + response = await page.goto(url, { + timeout: 15000, + waitUntil: 'domcontentloaded', + }); + } catch (navError) { + if (process.env.GOOGLE_SAFE_BROWSING) { + const errMsg = (navError as Error).message || ''; + consoleLogger.info(`[SafeBrowsing] Navigation error: ${errMsg}`); + if (errMsg.includes('ERR_BLOCKED_BY_CLIENT') || errMsg.includes('ERR_ABORTED')) { + consoleLogger.info('[SafeBrowsing] Page blocked by Chrome Safe Browsing!'); + const currentUrl = page.url(); + consoleLogger.info(`[SafeBrowsing] Current page URL after block: ${currentUrl}`); + } + } + throw navError; + } if (!response) throw new Error('No response from navigation'); + if (process.env.GOOGLE_SAFE_BROWSING) { + const navUrl = page.url(); + const navStatus = response.status(); + consoleLogger.info(`[SafeBrowsing] Navigation completed - status: ${navStatus}, url: ${navUrl}`); + if (navUrl.startsWith('chrome-error:') || navUrl.includes('interstitial')) { + consoleLogger.info('[SafeBrowsing] Interstitial/error page detected!'); + } + } + // Wait briefly for JS/meta-refresh redirects to settle before reading the final URL. // Server-side redirects are already reflected after goto(), but client-side redirects // (e.g. domain.tld -> www.domain.tld via JS or meta-refresh) need extra time. @@ -2216,10 +2242,29 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { ? ['--safebrowsing-disable-auto-update', '--disable-client-side-phishing-detection', '--disable-background-networking'] : []; + // Safe Browsing requires headful mode (Chrome doesn't enforce interstitials in headless). + // On Linux without a display, we start Xvfb to provide a virtual framebuffer. + let headless = process.env.CRAWLEE_HEADLESS === '1'; + if (safeBrowsingEnabled && headless) { + if (process.platform === 'linux' && !process.env.DISPLAY) { + try { + execSyncFn('pgrep -x Xvfb || (Xvfb :99 -screen 0 1024x768x24 &)', { stdio: 'ignore' }); + process.env.DISPLAY = ':99'; + consoleLogger.info('[SafeBrowsing] Started Xvfb on :99 for headful Safe Browsing'); + } catch { + consoleLogger.info('[SafeBrowsing] Failed to start Xvfb, falling back to headless'); + } + } + if (process.env.DISPLAY) { + headless = false; + consoleLogger.info(`[SafeBrowsing] Forcing headful mode (DISPLAY=${process.env.DISPLAY})`); + } + } + const options: LaunchOptions = { ignoreDefaultArgs: [...baseIgnoredArgs, ...safeBrowsingIgnoredArgs], args: finalArgs, - headless: process.env.CRAWLEE_HEADLESS === '1', + headless, ...(channel && { channel }), ...(proxyOpt ? { proxy: proxyOpt } : {}), }; @@ -2230,6 +2275,12 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { consoleLogger.info(`Enabled browser slowMo with value: ${process.env.OOBEE_SLOWMO}ms`); } + if (safeBrowsingEnabled) { + consoleLogger.info(`[SafeBrowsing] Launch options - ignoreDefaultArgs: ${JSON.stringify(options.ignoreDefaultArgs)}`); + consoleLogger.info(`[SafeBrowsing] Launch options - headless: ${options.headless}, channel: ${channel || 'bundled-chromium'}`); + consoleLogger.info(`[SafeBrowsing] Launch options - args: ${JSON.stringify(options.args)}`); + } + if (safeBrowsingEnabled && !!process.env.GOOGLE_SAFE_BROWSING_DEBUG) { options.args = [ ...(options.args ?? []), diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts index f589c97e..41faf300 100644 --- a/src/safeBrowsingProfile.ts +++ b/src/safeBrowsingProfile.ts @@ -164,12 +164,22 @@ async function spawnChromeForWarmup(): Promise { } export async function warmupSafeBrowsingBaseProfile(): Promise { - if (isDbDir(SB_DIR)) return; + consoleLogger.info(`[SafeBrowsing] BASE_PROFILE_DIR: ${BASE_PROFILE_DIR}`); + consoleLogger.info(`[SafeBrowsing] SB_DIR: ${SB_DIR}`); + consoleLogger.info(`[SafeBrowsing] isDbDir(SB_DIR): ${isDbDir(SB_DIR)}`); + if (isDbDir(SB_DIR)) { + consoleLogger.info('[SafeBrowsing] DB already exists in base profile, skipping warmup'); + return; + } fs.mkdirSync(BASE_PROFILE_DIR, { recursive: true }); const systemSbDir = findSystemSafeBrowsingDir(); + consoleLogger.info(`[SafeBrowsing] findSystemSafeBrowsingDir() = ${systemSbDir}`); if (systemSbDir) { + consoleLogger.info(`[SafeBrowsing] Found system DB at: ${systemSbDir}`); + const files = fs.readdirSync(systemSbDir); + consoleLogger.info(`[SafeBrowsing] Files in system DB: ${files.join(', ')}`); printMessage(['Copying Safe Browsing threat database from system Chrome profile...'], messageOptions); copyDirectory(systemSbDir, SB_DIR); printMessage(['Google Safe Browsing enabled (real-time URL protection active)'], messageOptions); @@ -178,11 +188,19 @@ export async function warmupSafeBrowsingBaseProfile(): Promise { const exe = getChromeExecutable(); const chromeFound = fs.existsSync(exe); + consoleLogger.info(`[SafeBrowsing] Chrome executable: ${exe}, found: ${chromeFound}`); if (!chromeFound) { printMessage(['WARNING: Google Chrome not found. Safe Browsing requires Chrome (not Chromium). On Linux Docker, build with --platform linux/amd64.'], messageOptions); return; } + // Chrome 128+ uses real-time v5 hash-prefix lookups via OHTTP and no longer + // downloads local UrlSoceng.store.* files. Skip the legacy DB download and + // rely on real-time API with preferences set by injectSafeBrowsingDb(). + consoleLogger.info('[SafeBrowsing] Skipping legacy DB download (Chrome 128+ uses real-time v5 API)'); + printMessage(['Google Safe Browsing enabled (real-time URL protection via Chrome)'], messageOptions); + return; + if (!acquireLock()) { consoleLogger.info('Another process is downloading Safe Browsing DB; waiting...'); const waitStart = Date.now(); @@ -206,9 +224,28 @@ export async function warmupSafeBrowsingBaseProfile(): Promise { } export function injectSafeBrowsingDb(targetDir: string): void { - if (!isDbDir(SB_DIR)) return; - if (fs.existsSync(path.join(targetDir, SEEDED_MARKER))) return; + consoleLogger.info(`[SafeBrowsing] injectSafeBrowsingDb(${targetDir})`); + consoleLogger.info(`[SafeBrowsing] isDbDir(SB_DIR=${SB_DIR}): ${isDbDir(SB_DIR)}`); + if (!isDbDir(SB_DIR)) { + consoleLogger.info('[SafeBrowsing] No DB to inject — setting preferences only'); + const defaultDir = path.join(targetDir, 'Default'); + fs.mkdirSync(defaultDir, { recursive: true }); + const prefsPath = path.join(defaultDir, 'Preferences'); + let prefs: Record = {}; + if (fs.existsSync(prefsPath)) { + try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} + } + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: true }; + fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + consoleLogger.info(`[SafeBrowsing] Wrote preferences to ${prefsPath}: ${JSON.stringify(prefs.safebrowsing)}`); + return; + } + if (fs.existsSync(path.join(targetDir, SEEDED_MARKER))) { + consoleLogger.info('[SafeBrowsing] Already seeded (marker exists), skipping'); + return; + } + consoleLogger.info('[SafeBrowsing] Copying DB + setting preferences'); copyDirectory(SB_DIR, path.join(targetDir, 'Safe Browsing')); const defaultDir = path.join(targetDir, 'Default'); @@ -220,12 +257,15 @@ export function injectSafeBrowsingDb(targetDir: string): void { } prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: true }; fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + consoleLogger.info(`[SafeBrowsing] Wrote preferences to ${prefsPath}: ${JSON.stringify(prefs.safebrowsing)}`); fs.writeFileSync(path.join(targetDir, SEEDED_MARKER), new Date().toISOString()); + consoleLogger.info('[SafeBrowsing] Injection complete'); } export async function ensureAndInjectSafeBrowsing(targetDir: string): Promise { if (!process.env.GOOGLE_SAFE_BROWSING) return; + consoleLogger.info(`[SafeBrowsing] ensureAndInjectSafeBrowsing(${targetDir})`); if (process.platform === 'win32') { printMessage(['Google Safe Browsing is not yet supported on Windows.'], messageOptions); @@ -234,4 +274,5 @@ export async function ensureAndInjectSafeBrowsing(targetDir: string): Promise Date: Fri, 10 Jul 2026 00:28:00 +0800 Subject: [PATCH 14/20] try push -enable-features=SafeBrowsingEnhancedProtection args --- src/constants/common.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/constants/common.ts b/src/constants/common.ts index b0701bc8..5ef67a20 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -2242,6 +2242,10 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { ? ['--safebrowsing-disable-auto-update', '--disable-client-side-phishing-detection', '--disable-background-networking'] : []; + if (safeBrowsingEnabled) { + finalArgs.push('--enable-features=SafeBrowsingEnhancedProtection'); + } + // Safe Browsing requires headful mode (Chrome doesn't enforce interstitials in headless). // On Linux without a display, we start Xvfb to provide a virtual framebuffer. let headless = process.env.CRAWLEE_HEADLESS === '1'; From d3e3c700bc61a84d8fbc1827498e0ebc6c3ee2cd Mon Sep 17 00:00:00 2001 From: Zui Young Date: Fri, 10 Jul 2026 00:30:49 +0800 Subject: [PATCH 15/20] use seed db method to test --- Dockerfile | 74 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 700e2e0c..3ace7535 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,19 +21,14 @@ RUN npx playwright install chromium # This is a Chrome-only feature — Chromium does NOT include it because it # requires Google's proprietary API keys baked into the Chrome build. # -# HOW IT WORKS (modern Chrome 128+): -# Chrome no longer downloads a local threat database (UrlSoceng.store.* files). -# Instead, it performs real-time hash-prefix lookups via the Safe Browsing v5 -# API using OHTTP (Oblivious HTTP) for privacy. This means: -# - No warmup/pre-seeding of a threat database is needed -# - Safe Browsing activates immediately on first navigation -# - The only requirements are: (1) Chrome (not Chromium), (2) safebrowsing -# enabled in Preferences, (3) network flags not suppressed +# HOW IT WORKS: +# Chrome needs a local hash-prefix database (UrlSoceng.store.*) to know which +# URLs require a real-time lookup via the v5 API. Without this DB, Chrome skips +# the check entirely. The DB is downloaded by Chrome when Safe Browsing is +# enabled and the browser runs with network access. # # ARCHITECTURE LIMITATION: # Google Chrome .deb packages are only available for amd64 (x86_64). -# As of July 2026, Google has announced ARM64 Linux Chrome but has not yet -# published it to their apt repository or direct download URL. # On arm64 builds, this step is skipped and Safe Browsing will not be available. # # TO ENABLE: Set env var GOOGLE_SAFE_BROWSING=1 when running the container. @@ -46,7 +41,52 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ fi -# --- App code (changes here don't invalidate Chrome layer above) --- +# Pre-seed Safe Browsing hash-prefix database into the image. +# Chrome needs Xvfb (virtual display) to run non-headless and download the DB. +# Under emulation this can take several minutes; timeout is set to 1800s. +COPY <<'SEEDSCRIPT' /tmp/seed-safe-browsing.sh +#!/bin/bash +set -e +mkdir -p /opt/oobee-safe-browsing/Default +echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > /opt/oobee-safe-browsing/Default/Preferences +export DISPLAY=:99 +Xvfb :99 -screen 0 1024x768x24 & +XVFB_PID=$! +sleep 2 +google-chrome \ + --user-data-dir=/opt/oobee-safe-browsing \ + --no-first-run --no-default-browser-check --disable-extensions \ + --no-sandbox --disable-setuid-sandbox \ + --enable-features=SafeBrowsingEnhancedProtection \ + --window-position=-10000,-10000 --window-size=1,1 \ + about:blank & +CHROME_PID=$! +echo "Waiting for Safe Browsing DB to download (up to 1800s)..." +WAITED=0 +while [ $WAITED -lt 1800 ]; do + if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1 || \ + ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlMalware.store.* >/dev/null 2>&1; then + echo "Safe Browsing DB downloaded successfully (${WAITED}s)" + break + fi + sleep 10 + WAITED=$((WAITED + 10)) + echo "[${WAITED}s] Still waiting... $(ls /opt/oobee-safe-browsing/Safe\ Browsing/ 2>/dev/null | wc -l) files in Safe Browsing dir" +done +kill $CHROME_PID 2>/dev/null || true +kill $XVFB_PID 2>/dev/null || true +if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1; then + echo "Safe Browsing DB baked into image" +else + echo "WARNING: Safe Browsing DB did not populate - will attempt at runtime" +fi +SEEDSCRIPT +RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome >/dev/null 2>&1; then \ + bash /tmp/seed-safe-browsing.sh; \ + fi && rm -f /tmp/seed-safe-browsing.sh && \ + chmod -R a+rX /opt/oobee-safe-browsing 2>/dev/null || true + +# --- App code (changes here don't invalidate Chrome/seed layers above) --- WORKDIR /app/oobee @@ -63,21 +103,15 @@ COPY . . RUN npm run build || true # true exits with code 0 - workaround for TS errors # Add non-privileged user -# Create a group named "purple" -RUN groupadd -r purple - -# Create a user named "purple" and assign it to the group "purple" -RUN useradd -r -g purple purple - -# Create a dedicated directory for the "purple" user and set permissions -RUN mkdir -p /home/purple && chown -R purple:purple /home/purple +RUN groupadd -r purple && useradd -r -g purple purple && \ + mkdir -p /home/purple && chown -R purple:purple /home/purple WORKDIR /app # Set the ownership of the oobee directory to the user "purple" RUN chown -R purple:purple /app -# For oobee to be run from present working directory, comment out as necessary +# For oobee to be run from present working directory WORKDIR /app/oobee # Run everything after as non-privileged user. From 5d94c5d12b61fe63d504157572c4e46293bbd9bf Mon Sep 17 00:00:00 2001 From: Zui Young Date: Tue, 14 Jul 2026 06:42:18 +0000 Subject: [PATCH 16/20] test: add qemu emulation support for chrome launch --- src/constants/common.ts | 132 ++++++++++++++++++++++++++++++++++------ 1 file changed, 112 insertions(+), 20 deletions(-) diff --git a/src/constants/common.ts b/src/constants/common.ts index 5ef67a20..9bdd967e 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -309,6 +309,30 @@ const isAllowedContentType = (ct: string): boolean => { ); }; +const isLikelyQemuChromeCrash = (error: unknown): boolean => { + const msg = error instanceof Error ? error.message : String(error || ''); + return ( + msg.includes('Target page, context or browser has been closed') || + msg.includes("GPU process isn't usable") || + msg.includes('qemu: uncaught target signal') + ); +}; + +const withQemuChromeWorkaroundArgs = (launchOptions: LaunchOptions): LaunchOptions => { + const args = [...(launchOptions.args ?? [])]; + const extraArgs = [ + '--in-process-gpu', + '--disable-gpu-compositing', + '--disable-features=VizDisplayCompositor', + ]; + + extraArgs.forEach(arg => { + if (!args.includes(arg)) args.push(arg); + }); + + return { ...launchOptions, args }; +}; + const checkUrlConnectivityWithBrowser = async ( url: string, browserToRun: string, @@ -404,32 +428,63 @@ const checkUrlConnectivityWithBrowser = async ( // Keep UA emulation explicitly. contextOptions.userAgent = process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined); - try { - const launchPersistent = async () => { - browserContext = await launchPersistentSafeContext(clonedDataDir, { - ...launchOptions, - ...contextOptions, - }); - register(browserContext); - }; + const launchPersistent = async (effectiveLaunchOptions: LaunchOptions) => { + browserContext = await launchPersistentSafeContext(clonedDataDir, { + ...effectiveLaunchOptions, + ...contextOptions, + }); + register(browserContext); + }; - const launchEphemeral = async () => { - browserInstance = await constants.launcher.launch(launchOptions); - register(browserInstance as unknown as { close: () => Promise }); - browserContext = await browserInstance.newContext(contextOptions); - }; + const launchEphemeral = async (effectiveLaunchOptions: LaunchOptions) => { + browserInstance = await constants.launcher.launch(effectiveLaunchOptions); + register(browserInstance as unknown as { close: () => Promise }); + browserContext = await browserInstance.newContext(contextOptions); + }; + const launchBrowserContext = async (effectiveLaunchOptions: LaunchOptions) => { if (process.env.CRAWLEE_HEADLESS === '1') { try { - await launchPersistent(); + await launchPersistent(effectiveLaunchOptions); } catch (error) { // Fallback to ephemeral context if persistent context fails (e.g. protocol errors) // More prone to falling back here when running localFile scans consoleLogger.warn(`Persistent context launch failed, retrying with ephemeral context: ${error.message}`); - await launchEphemeral(); + await launchEphemeral(effectiveLaunchOptions); } } else { - await launchEphemeral(); + await launchEphemeral(effectiveLaunchOptions); + } + }; + + try { + try { + await launchBrowserContext(launchOptions); + } catch (error) { + const shouldRetryWithQemuArgs = + browserToRun === BrowserTypes.CHROME && + process.platform === 'linux' && + fs.existsSync('/.dockerenv') && + isLikelyQemuChromeCrash(error); + + if (!shouldRetryWithQemuArgs) throw error; + + consoleLogger.warn( + '[Chrome] Launch failed with likely emulation/GPU issue. Retrying Chrome with QEMU-safe flags.', + ); + + try { + await browserContext?.close(); + } catch {} + if (browserInstance) { + try { + await browserInstance.close(); + } catch {} + } + browserContext = undefined; + browserInstance = undefined; + + await launchBrowserContext(withQemuChromeWorkaroundArgs(launchOptions)); } } catch (err) { printMessage([`Unable to launch browser\n${err}`], messageOptions); @@ -2151,11 +2206,12 @@ export async function initModifiedUserAgent( ) { // UA bootstrap must not use persistent context / user-data-dir. const launchOptions = getPlaywrightLaunchOptions(browser); - - const browserInstance = await constants.launcher.launch(launchOptions); - register(browserInstance as unknown as { close: () => Promise }); + let browserInstance: Awaited> | undefined; try { + browserInstance = await constants.launcher.launch(launchOptions); + register(browserInstance as unknown as { close: () => Promise }); + const context = await browserInstance.newContext(); const page = await context.newPage(); const defaultUA = await page.evaluate(() => navigator.userAgent); @@ -2167,8 +2223,21 @@ export async function initModifiedUserAgent( // Do not mutate global CLI args with --user-agent= process.env.OOBEE_USER_AGENT = modifiedUA; + } catch (error) { + const fallbackUA = + (typeof (_playwrightDeviceDetailsObject as any)?.userAgent === 'string' && + (_playwrightDeviceDetailsObject as any).userAgent) || + devices['Desktop Chrome'].userAgent; + + process.env.OOBEE_USER_AGENT = fallbackUA.includes('HeadlessChrome') + ? fallbackUA.replace('HeadlessChrome', 'Chrome') + : fallbackUA; + + consoleLogger.warn( + `[UA] Failed to bootstrap user agent from browser (${(error as Error).message}). Using fallback Chrome UA string.`, + ); } finally { - await browserInstance.close(); + await browserInstance?.close(); } } @@ -2188,6 +2257,8 @@ export async function launchPersistentSafeContext( */ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { const channel = browser || undefined; + const isLinuxDockerChrome = + browser === BrowserTypes.CHROME && process.platform === 'linux' && fs.existsSync('/.dockerenv'); const resolution = proxyInfoToResolution(cacheProxyInfo); const shouldIgnoreMuteAudio = @@ -2206,6 +2277,23 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { // during long crawls with multiple pool rotations finalArgs.push('--disk-cache-size=10485760'); + // Chrome under Linux Docker emulation can crash in GPU process startup. + // Apply a conservative software-only launch profile while keeping Chrome channel. + if (isLinuxDockerChrome) { + const dockerChromeArgs = [ + '--in-process-gpu', + '--disable-gpu-compositing', + '--disable-software-rasterizer', + '--disable-features=VizDisplayCompositor', + '--single-process', + '--no-zygote', + ]; + + dockerChromeArgs.forEach(arg => { + if (!finalArgs.includes(arg)) finalArgs.push(arg); + }); + } + // Prevent Windows from throttling background Chromium processes if (os.platform() === 'win32') { finalArgs.push('--disable-features=UseEcoQoSForBackgroundProcess'); @@ -2238,6 +2326,10 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { ? ['--use-mock-keychain', '--mute-audio'] : ['--use-mock-keychain']; + if (isLinuxDockerChrome && !baseIgnoredArgs.includes('--enable-unsafe-swiftshader')) { + baseIgnoredArgs.push('--enable-unsafe-swiftshader'); + } + const safeBrowsingIgnoredArgs = safeBrowsingEnabled ? ['--safebrowsing-disable-auto-update', '--disable-client-side-phishing-detection', '--disable-background-networking'] : []; From ac64e2782fa96594b80ad377a086173dc582f74f Mon Sep 17 00:00:00 2001 From: Zui Young Date: Tue, 14 Jul 2026 06:43:26 +0000 Subject: [PATCH 17/20] perf(docker): remove recursive chown layer via COPY --chown --- Dockerfile | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3ace7535..4e08293a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,31 +88,23 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome # --- App code (changes here don't invalidate Chrome/seed layers above) --- -WORKDIR /app/oobee - # Environment variables for node and Playwright ENV NODE_ENV=production ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="true" -# Install dependencies first (cached unless package.json/package-lock.json change) -COPY package.json package-lock.json ./ -RUN npm install --omit=dev - -# Copy source and compile TypeScript -COPY . . -RUN npm run build || true # true exits with code 0 - workaround for TS errors - -# Add non-privileged user +# Add non-privileged user before app copy so ownership can be set during COPY. RUN groupadd -r purple && useradd -r -g purple purple && \ - mkdir -p /home/purple && chown -R purple:purple /home/purple + mkdir -p /home/purple /app/oobee && chown purple:purple /home/purple /app /app/oobee -WORKDIR /app - -# Set the ownership of the oobee directory to the user "purple" -RUN chown -R purple:purple /app - -# For oobee to be run from present working directory WORKDIR /app/oobee -# Run everything after as non-privileged user. +# Run app build steps as non-root to avoid a full recursive chown later. USER purple + +# Install dependencies first (cached unless package.json/package-lock.json change) +COPY --chown=purple:purple package.json package-lock.json ./ +RUN npm install --omit=dev + +# Copy source and compile TypeScript +COPY --chown=purple:purple . . +RUN npm run build || true # true exits with code 0 - workaround for TS errors From 1a2ee3f5c3d86aae6e8b8a53b2663ffb0f7f8367 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Tue, 14 Jul 2026 07:16:07 +0000 Subject: [PATCH 18/20] test: harden safe browsing xvfb/display startup in container --- src/constants/common.ts | 71 +++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/src/constants/common.ts b/src/constants/common.ts index 9bdd967e..07427300 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -14,7 +14,7 @@ import url, { fileURLToPath, pathToFileURL } from 'url'; import safe from 'safe-regex'; import * as https from 'https'; import os from 'os'; -import { execSync as execSyncFn } from 'child_process'; +import { execSync as execSyncFn, spawn as spawnFn } from 'child_process'; import mime from 'mime'; import { minimatch } from 'minimatch'; import { globSync, GlobOptionsWithFileTypesFalse } from 'glob'; @@ -333,6 +333,42 @@ const withQemuChromeWorkaroundArgs = (launchOptions: LaunchOptions): LaunchOptio return { ...launchOptions, args }; }; +const ensureXvfbDisplay = (): string | null => { + if (process.platform !== 'linux') return null; + + const candidates = [process.env.DISPLAY, ':99', ':98', ':97', ':96'] + .filter((v): v is string => typeof v === 'string' && v.length > 0) + .filter((v, i, arr) => arr.indexOf(v) === i); + + for (const display of candidates) { + const displayNum = display.replace(':', ''); + const socketPath = `/tmp/.X11-unix/X${displayNum}`; + + // Reuse an already-live display socket. + if (fs.existsSync(socketPath)) return display; + + try { + const xvfb = spawnFn('Xvfb', [display, '-screen', '0', '1024x768x24', '-nolisten', 'tcp'], { + detached: true, + stdio: 'ignore', + }); + xvfb.unref(); + + // Wait briefly for the X socket to become available. + execSyncFn( + `for i in 1 2 3 4 5 6 7 8 9 10; do [ -S ${socketPath} ] && exit 0; sleep 0.2; done; exit 1`, + { stdio: 'ignore' }, + ); + + return display; + } catch { + // Try next display candidate. + } + } + + return null; +}; + const checkUrlConnectivityWithBrowser = async ( url: string, browserToRun: string, @@ -2285,7 +2321,6 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { '--disable-gpu-compositing', '--disable-software-rasterizer', '--disable-features=VizDisplayCompositor', - '--single-process', '--no-zygote', ]; @@ -2342,13 +2377,22 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { // On Linux without a display, we start Xvfb to provide a virtual framebuffer. let headless = process.env.CRAWLEE_HEADLESS === '1'; if (safeBrowsingEnabled && headless) { - if (process.platform === 'linux' && !process.env.DISPLAY) { - try { - execSyncFn('pgrep -x Xvfb || (Xvfb :99 -screen 0 1024x768x24 &)', { stdio: 'ignore' }); - process.env.DISPLAY = ':99'; - consoleLogger.info('[SafeBrowsing] Started Xvfb on :99 for headful Safe Browsing'); - } catch { - consoleLogger.info('[SafeBrowsing] Failed to start Xvfb, falling back to headless'); + if (process.platform === 'linux') { + const currentDisplay = process.env.DISPLAY; + const currentDisplayNum = (currentDisplay || '').replace(':', ''); + const currentDisplaySocket = currentDisplayNum + ? `/tmp/.X11-unix/X${currentDisplayNum}` + : ''; + + // Recover from stale DISPLAY values inherited from shell sessions. + if (!currentDisplay || !currentDisplaySocket || !fs.existsSync(currentDisplaySocket)) { + const display = ensureXvfbDisplay(); + if (display) { + process.env.DISPLAY = display; + consoleLogger.info(`[SafeBrowsing] Xvfb ready on ${display} for headful Safe Browsing`); + } else { + consoleLogger.info('[SafeBrowsing] Failed to start Xvfb, falling back to headless'); + } } } if (process.env.DISPLAY) { @@ -2365,6 +2409,15 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { ...(proxyOpt ? { proxy: proxyOpt } : {}), }; + // Be explicit so Chrome child processes always inherit DISPLAY when + // Safe Browsing forces headed mode in Linux containers. + if (safeBrowsingEnabled && process.platform === 'linux' && process.env.DISPLAY) { + options.env = { + ...process.env, + DISPLAY: process.env.DISPLAY, + } as NodeJS.ProcessEnv; + } + // SlowMo for debugging, can be set via env variable OOBEE_SLOWMO to avoid adding it as a CLI argument and causing confusion for users who don't need it if (!options.slowMo && process.env.OOBEE_SLOWMO && Number(process.env.OOBEE_SLOWMO) >= 1) { options.slowMo = Number(process.env.OOBEE_SLOWMO); From df5745badd02cfaae367ab23d4cee205547a96ee Mon Sep 17 00:00:00 2001 From: Zui Young Date: Tue, 14 Jul 2026 07:34:27 +0000 Subject: [PATCH 19/20] fix(safebrowsing): allow component updates in linux docker run --- src/constants/common.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/constants/common.ts b/src/constants/common.ts index 07427300..29fbaccb 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -2366,7 +2366,12 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { } const safeBrowsingIgnoredArgs = safeBrowsingEnabled - ? ['--safebrowsing-disable-auto-update', '--disable-client-side-phishing-detection', '--disable-background-networking'] + ? [ + '--safebrowsing-disable-auto-update', + '--disable-client-side-phishing-detection', + '--disable-background-networking', + '--disable-component-update', + ] : []; if (safeBrowsingEnabled) { From 7a650de4194dc1a6d1b7005c2dbbf7c081068468 Mon Sep 17 00:00:00 2001 From: Zui Young Date: Mon, 20 Jul 2026 14:12:58 +0800 Subject: [PATCH 20/20] feat: Google Safe Browsing URL protection for all scan types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds optional Safe Browsing support gated by GOOGLE_SAFE_BROWSING env var. Blocks phishing, malware, and unwanted software URLs — blocked pages are classified as "Blocked by Safe Browsing" in reports. Platform behavior: - macOS: copies Safe Browsing threat DB from system Chrome profile for immediate local detection - Docker Linux: connects to pre-warmed Chrome via CDP (port 9222) which has an active OHTTP relay - runCustom: uses persistent context on macOS (with system DB), CDP on Docker; shows interstitial warning to users Detection: - chrome-error:// pages in request handler → "Blocked by Safe Browsing" - ERR_BLOCKED_BY_CLIENT/RESPONSE in failed request handler → same - urlGuard allowChromeErrors lets interstitial stay visible in runCustom Launch options (when GOOGLE_SAFE_BROWSING set): - Removes flags that suppress SB (--safebrowsing-disable-auto-update, --disable-client-side-phishing-detection, --disable-background-networking, --disable-component-update) - Forces headful mode (interstitials don't render in headless) - Adds --enable-features=SafeBrowsingEnhancedProtection Without GOOGLE_SAFE_BROWSING: zero behavior change, no overhead. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 51 ++++--- Dockerfile | 84 ++++------- README.md | 2 +- gsb-test/start-gsb-novnc.sh | 93 ++++++++++++ src/constants/common.ts | 232 +++++++++++++++--------------- src/constants/constants.ts | 10 ++ src/crawlers/commonCrawlerFunc.ts | 19 ++- src/crawlers/crawlDomain.ts | 38 ++++- src/crawlers/crawlSitemap.ts | 32 ++++- src/crawlers/custom/utils.ts | 31 +++- src/crawlers/runCustom.ts | 46 ++++-- 11 files changed, 409 insertions(+), 229 deletions(-) create mode 100644 gsb-test/start-gsb-novnc.sh diff --git a/AGENTS.md b/AGENTS.md index 041cf9d1..f30e25df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,10 +147,7 @@ The `constants` default export object holds runtime state: <<<<<<< HEAD | `OOBEE_SAVE_DOM` | `1` or `true` = save full-page DOM HTML to `pageDOMs/` in results directory. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | | `OOBEE_SAVE_PAGE_SCREENSHOT` | `1` or `true` = save full-page desktop + mobile viewport screenshots to `pageDOMs/desktopPageScreenshots/` and `pageDOMs/mobilePageScreenshots/`. Mobile viewport uses iPhone 11 width programmatically. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | -======= | `GOOGLE_SAFE_BROWSING` | `1` = enable Google Safe Browsing (requires Chrome, not Chromium) | -| `GOOGLE_SAFE_BROWSING_DEBUG` | `1` = enable verbose Safe Browsing debug logging | ->>>>>>> 86d9da36 (feat: add Google Safe Browsing support via Chrome real-time URL protection) | `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` | Proxy configuration | | `NO_PROXY` / `INCLUDE_PROXY` | Proxy bypass/include lists | @@ -189,52 +186,60 @@ The `constants` default export object holds runtime state: ### Overview -Google Safe Browsing protects users by blocking navigation to phishing/malware URLs. It requires the local hash-prefix threat database (`UrlSoceng.store.*`, `UrlMalware.store.*`) to be present in the browser profile. Chrome uses this local DB as a first-pass filter and shows interstitial warning pages when a match is found. +Google Safe Browsing protects users by blocking navigation to phishing/malware URLs. On macOS, it copies the local hash-prefix threat database (`UrlSoceng.store.*`, `UrlMalware.store.*`) from the system Chrome profile. On Docker Linux, it connects via CDP to a pre-warmed Chrome instance with an active OHTTP relay. Blocked pages are classified as "Blocked by Safe Browsing" in scan results. ### Requirements -1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys. Chromium does not include them. +1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys. 2. **`GOOGLE_SAFE_BROWSING`** environment variable set to any value. 3. **macOS or Linux only** — Windows is not yet supported (prints a warning and skips). ### How It Works -When `GOOGLE_SAFE_BROWSING` is set: +#### macOS (all scan types) -1. `ensureAndInjectSafeBrowsing()` in `src/safeBrowsingProfile.ts` runs before each browser launch: - - **Fast path**: Copies the threat DB from your system Chrome profile (`~/Library/Application Support/Google/Chrome/Safe Browsing` on macOS, `~/.config/google-chrome/Safe Browsing` on Linux). This is instant. - - **Slow path**: If no system profile exists, spawns a real Chrome process for up to 120s to download the DB. On Linux without DISPLAY, tries Xvfb first, falls back to `--headless=old`. - - Writes `safebrowsing: { enabled: true }` into the profile's Preferences. - - Uses a file lock (`~/.oobee/safe-browsing-profile/.warmup-lock`) to prevent concurrent processes from corrupting the DB. +1. `ensureAndInjectSafeBrowsing()` copies the Safe Browsing DB from your system Chrome profile (`~/Library/Application Support/Google/Chrome/Safe Browsing/`) into a base profile at `~/.oobee/safe-browsing-profile/`. +2. `injectSafeBrowsingDb()` copies the DB into each scan's browser profile directory and writes `safebrowsing: { enabled: true, enhanced: true }` into Preferences. +3. `getPlaywrightLaunchOptions()` removes Playwright default flags that suppress Safe Browsing (`--safebrowsing-disable-auto-update`, `--disable-client-side-phishing-detection`, `--disable-background-networking`, `--disable-component-update`) via `ignoreDefaultArgs`. +4. Chrome uses the local DB for immediate threat detection without needing the OHTTP relay. -2. `getPlaywrightLaunchOptions()` in `src/constants/common.ts` adds three Playwright default args to `ignoreDefaultArgs`: - - `--safebrowsing-disable-auto-update` - - `--disable-background-networking` - - `--disable-client-side-phishing-detection` +#### Docker Linux (crawlDomain, crawlSitemap) -3. `urlGuard.ts` allows `chrome-error://` protocol when Safe Browsing is active, so the interstitial warning page is not redirected away. +1. A pre-warmed Chrome runs on port 9222 (started by `start-gsb-novnc.sh`) with an active OHTTP relay. +2. `getSafeBrowsingCdpLauncher()` connects to this Chrome via CDP and returns the default context (which has active Safe Browsing). +3. Crawlee's browser pool uses this CDP launcher instead of spawning fresh instances. + +#### Docker Linux (runCustom) + +1. Same CDP approach — `getSafeBrowsingCdpLauncher()` connects to the pre-warmed Chrome. +2. Falls back to `launchPersistentSafeContext()` if CDP is unavailable. + +#### Detection of blocked pages + +- **requestHandler**: Pages that navigate to `chrome-error://` are classified as "Blocked by Safe Browsing" (`STATUS_CODE_METADATA[3]`). +- **failedRequestHandler**: Navigation errors containing `ERR_BLOCKED_BY_CLIENT` or `ERR_BLOCKED_BY_RESPONSE` are classified as "Blocked by Safe Browsing". +- **runCustom**: `urlGuard.ts` with `allowChromeErrors: true` lets the interstitial page stay visible to the user. ### Docker / Architecture Constraints - **amd64 (x86_64)**: Google Chrome .deb is installed in Dockerfile. Safe Browsing works fully. - **arm64 (aarch64)**: Chrome is NOT available for ARM64 Linux. Safe Browsing is unavailable. - Build the image with `docker build --platform linux/amd64` to ensure Chrome is available. -- On first run in Docker, the slow path (spawning Chrome) is used since there's no system profile. ### Environment Variables | Variable | Purpose | |----------|---------| | `GOOGLE_SAFE_BROWSING` | Enable Safe Browsing (any value; requires Chrome) | -| `GOOGLE_SAFE_BROWSING_DEBUG` | Enable verbose Chrome Safe Browsing logging | ### Key Files -- `src/safeBrowsingProfile.ts` — DB warmup, injection, and seeding logic -- `src/constants/common.ts` — `getPlaywrightLaunchOptions()` (ignoreDefaultArgs), `launchPersistentSafeContext()` wrapper -- `src/crawlers/guards/urlGuard.ts` — `allowChromeErrors` for interstitial pages -- `src/crawlers/runCustom.ts` — passes `allowChromeErrors` to urlGuard -- `Dockerfile` — Conditional Chrome installation for amd64 +- `src/safeBrowsingProfile.ts` — DB warmup, copy from system profile, injection into scan profiles +- `src/constants/common.ts` — `getSafeBrowsingCdpLauncher()`, `getPlaywrightLaunchOptions()` (ignoreDefaultArgs), `launchPersistentSafeContext()` +- `src/crawlers/guards/urlGuard.ts` — `allowChromeErrors` for interstitial pages in runCustom +- `src/crawlers/crawlDomain.ts` — `chrome-error:` and `ERR_BLOCKED_BY_CLIENT` detection +- `src/crawlers/crawlSitemap.ts` — same blocked-page detection +- `src/crawlers/runCustom.ts` — CDP on Docker, persistent context on macOS ### What Does NOT Work diff --git a/Dockerfile b/Dockerfile index 4e08293a..a737219f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,19 @@ # Node version is v22 FROM mcr.microsoft.com/playwright:v1.61.1-noble + # Installation of packages for oobee RUN apt-get update && apt-get install -y --no-install-recommends \ git \ unzip \ zip \ - xvfb && \ + xvfb \ + x11vnc \ + novnc \ + websockify \ + openbox \ + procps \ + libnss3-tools && \ rm -rf /var/lib/apt/lists/* # Install Playwright browsers @@ -16,20 +23,13 @@ RUN npx playwright install chromium # ============================================================================= # Google Chrome installation for Safe Browsing support # ============================================================================= -# WHY: Chrome's Safe Browsing (v5, hash-real-time protocol) protects users by -# checking URLs against Google's threat database in real-time via OHTTP. -# This is a Chrome-only feature — Chromium does NOT include it because it -# requires Google's proprietary API keys baked into the Chrome build. -# -# HOW IT WORKS: -# Chrome needs a local hash-prefix database (UrlSoceng.store.*) to know which -# URLs require a real-time lookup via the v5 API. Without this DB, Chrome skips -# the check entirely. The DB is downloaded by Chrome when Safe Browsing is -# enabled and the browser runs with network access. +# WHY: Chrome's Safe Browsing (v5) checks URLs in real-time via Google's OHTTP +# relay — no local hash database is needed or pre-seeded. +# This is Chrome-only; Chromium lacks the required proprietary API keys. # # ARCHITECTURE LIMITATION: -# Google Chrome .deb packages are only available for amd64 (x86_64). -# On arm64 builds, this step is skipped and Safe Browsing will not be available. +# Chrome .deb packages are only available for amd64 (x86_64). +# On arm64 builds this step is skipped and Safe Browsing will not be available. # # TO ENABLE: Set env var GOOGLE_SAFE_BROWSING=1 when running the container. # ============================================================================= @@ -41,52 +41,20 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ fi -# Pre-seed Safe Browsing hash-prefix database into the image. -# Chrome needs Xvfb (virtual display) to run non-headless and download the DB. -# Under emulation this can take several minutes; timeout is set to 1800s. -COPY <<'SEEDSCRIPT' /tmp/seed-safe-browsing.sh -#!/bin/bash -set -e -mkdir -p /opt/oobee-safe-browsing/Default -echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > /opt/oobee-safe-browsing/Default/Preferences -export DISPLAY=:99 -Xvfb :99 -screen 0 1024x768x24 & -XVFB_PID=$! -sleep 2 -google-chrome \ - --user-data-dir=/opt/oobee-safe-browsing \ - --no-first-run --no-default-browser-check --disable-extensions \ - --no-sandbox --disable-setuid-sandbox \ - --enable-features=SafeBrowsingEnhancedProtection \ - --window-position=-10000,-10000 --window-size=1,1 \ - about:blank & -CHROME_PID=$! -echo "Waiting for Safe Browsing DB to download (up to 1800s)..." -WAITED=0 -while [ $WAITED -lt 1800 ]; do - if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1 || \ - ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlMalware.store.* >/dev/null 2>&1; then - echo "Safe Browsing DB downloaded successfully (${WAITED}s)" - break - fi - sleep 10 - WAITED=$((WAITED + 10)) - echo "[${WAITED}s] Still waiting... $(ls /opt/oobee-safe-browsing/Safe\ Browsing/ 2>/dev/null | wc -l) files in Safe Browsing dir" -done -kill $CHROME_PID 2>/dev/null || true -kill $XVFB_PID 2>/dev/null || true -if ls /opt/oobee-safe-browsing/Safe\ Browsing/UrlSoceng.store.* >/dev/null 2>&1; then - echo "Safe Browsing DB baked into image" -else - echo "WARNING: Safe Browsing DB did not populate - will attempt at runtime" -fi -SEEDSCRIPT -RUN if [ "$(dpkg --print-architecture)" = "amd64" ] && command -v google-chrome >/dev/null 2>&1; then \ - bash /tmp/seed-safe-browsing.sh; \ - fi && rm -f /tmp/seed-safe-browsing.sh && \ - chmod -R a+rX /opt/oobee-safe-browsing 2>/dev/null || true -# --- App code (changes here don't invalidate Chrome/seed layers above) --- +# --- noVNC / Safe Browsing warmup entrypoint --- +# Baked into the base image so the same oobee image can be used as a GUI +# test container by overriding the entrypoint: +# docker run ... -p 6080:6080 oobee /usr/local/bin/start-gsb-novnc.sh +COPY gsb-test/start-gsb-novnc.sh /usr/local/bin/start-gsb-novnc.sh +RUN chmod +x /usr/local/bin/start-gsb-novnc.sh +ENV GSB_WARMUP_SECONDS=300 +ENV GSB_WARMUP_URL=https://example.com +ENV GOOGLE_SAFE_BROWSING=1 +EXPOSE 6080 5900 +VOLUME ["/data/chrome-profile"] + +# --- App code (changes here don't invalidate Chrome layers above) --- # Environment variables for node and Playwright ENV NODE_ENV=production diff --git a/README.md b/README.md index 2effef6e..811e7d84 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ verapdf --version | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | | | NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | | | INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | | -| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing URL protection. Requires Google Chrome (not Chromium) to be installed. Seeds the Safe Browsing threat database from your system Chrome profile on first run. macOS and Linux only (Windows not yet supported). | | +| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing URL protection. Blocks phishing, malware, and unwanted software URLs — blocked pages are classified as "Blocked by Safe Browsing" in reports. Requires Google Chrome (not Chromium). On macOS, copies the threat database from your system Chrome profile. On Docker Linux, connects to a pre-warmed Chrome instance. macOS and Linux only (Windows not yet supported). | | #### Environment variables used internally (Do not set) Do not set these environment variables or behaviour might change unexpectedly. diff --git a/gsb-test/start-gsb-novnc.sh b/gsb-test/start-gsb-novnc.sh new file mode 100644 index 00000000..87aef96a --- /dev/null +++ b/gsb-test/start-gsb-novnc.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +export DISPLAY=${DISPLAY:-:99} +PROFILE_DIR=${PROFILE_DIR:-/data/chrome-profile} +export OOBEE_CHROME_DATA_DIR="$PROFILE_DIR" +WARMUP_SECONDS=${GSB_WARMUP_SECONDS:-300} +CHROME_URL=${GSB_WARMUP_URL:-https://example.com} + +mkdir -p "$PROFILE_DIR/Default" + +# Import Cloudflare CA into Chrome's NSS database so HTTPS interception +# by corporate proxies doesn't trigger "connection is not private" errors. +NSS_DB="$PROFILE_DIR/Default" +if [ -f /usr/local/share/ca-certificates/Cloudflare_CA.crt ] && command -v certutil >/dev/null 2>&1; then + mkdir -p "$NSS_DB" + if [ ! -d "$NSS_DB" ] || ! certutil -L -d sql:"$NSS_DB" -n "Cloudflare CA" >/dev/null 2>&1; then + certutil -d sql:"$NSS_DB" -N --empty-password 2>/dev/null || true + certutil -d sql:"$NSS_DB" -A -t "C,," -n "Cloudflare CA" \ + -i /usr/local/share/ca-certificates/Cloudflare_CA.crt + echo "[GSB] Cloudflare CA imported into Chrome NSS database" + fi +fi + +# Ensure Safe Browsing is enabled in profile Preferences on first run. +if [ ! -f "$PROFILE_DIR/Default/Preferences" ]; then + echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > "$PROFILE_DIR/Default/Preferences" +fi + +rm -f /tmp/.X99-lock +Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp >/tmp/xvfb.log 2>&1 & +openbox >/tmp/openbox.log 2>&1 & +x11vnc -display :99 -forever -shared -nopw -listen 0.0.0.0 -rfbport 5900 >/tmp/x11vnc.log 2>&1 & + +# Start Chrome and keep it alive in the background for the lifetime of the container. +# This keeps the Safe Browsing real-time OHTTP relay connection warm so the first +# Oobee scan does not have to wait for the ~5 minute cold-start initialization. +CHROME_ARGS=( + --user-data-dir="$PROFILE_DIR" + --profile-directory=Default + --no-sandbox --disable-setuid-sandbox + --disable-dev-shm-usage --disable-gpu --disable-software-rasterizer + --in-process-gpu --disable-gpu-compositing --disable-features=VizDisplayCompositor + --no-zygote --no-first-run --no-default-browser-check + --enable-features=SafeBrowsingEnhancedProtection + --ignore-certificate-errors + --ozone-platform=x11 + --remote-debugging-port=9222 +) + +if command -v google-chrome >/dev/null 2>&1; then + rm -f "$PROFILE_DIR/SingletonLock" "$PROFILE_DIR/SingletonCookie" "$PROFILE_DIR/SingletonSocket" + google-chrome "${CHROME_ARGS[@]}" "$CHROME_URL" >/tmp/chrome-warmup.log 2>&1 & + CHROME_PID=$! + echo "[GSB] Chrome started (PID $CHROME_PID) — waiting for OHTTP key (max ${WARMUP_SECONDS}s)..." + + WAITED=0 + ATTEMPT=0 + GSB_READY=0 + MAX_ATTEMPTS=$(( WARMUP_SECONDS / 5 )) + while [ "$WAITED" -lt "$WARMUP_SECONDS" ]; do + sleep 5 + WAITED=$((WAITED + 5)) + ATTEMPT=$((ATTEMPT + 1)) + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — checking for OHTTP key (${WAITED}s elapsed)..." + KEY=$(python3 -c " +import json, sys +try: + with open('$PROFILE_DIR/Default/Preferences') as f: + d = json.load(f) + print(d.get('safebrowsing', {}).get('hash_real_time_ohttp_key', '')) +except Exception: + print('') +" 2>/dev/null || true) + if [ -n "$KEY" ]; then + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — OHTTP key present after ${WAITED}s — Safe Browsing READY" + GSB_READY=1 + break + fi + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — key not yet present, will retry in 5s" + done + + if [ "$GSB_READY" -eq 0 ]; then + echo "[GSB] WARNING: OHTTP key not found after ${WARMUP_SECONDS}s — Safe Browsing may not be fully ready" + fi + echo "[GSB] Chrome remains running (PID $CHROME_PID)" +fi + +if [ -x /usr/share/novnc/utils/novnc_proxy ]; then + exec /usr/share/novnc/utils/novnc_proxy --listen 6080 --vnc localhost:5900 +else + exec websockify --web=/usr/share/novnc 6080 localhost:5900 +fi diff --git a/src/constants/common.ts b/src/constants/common.ts index 29fbaccb..b8c76d70 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -14,7 +14,7 @@ import url, { fileURLToPath, pathToFileURL } from 'url'; import safe from 'safe-regex'; import * as https from 'https'; import os from 'os'; -import { execSync as execSyncFn, spawn as spawnFn } from 'child_process'; + import mime from 'mime'; import { minimatch } from 'minimatch'; import { globSync, GlobOptionsWithFileTypesFalse } from 'glob'; @@ -333,42 +333,6 @@ const withQemuChromeWorkaroundArgs = (launchOptions: LaunchOptions): LaunchOptio return { ...launchOptions, args }; }; -const ensureXvfbDisplay = (): string | null => { - if (process.platform !== 'linux') return null; - - const candidates = [process.env.DISPLAY, ':99', ':98', ':97', ':96'] - .filter((v): v is string => typeof v === 'string' && v.length > 0) - .filter((v, i, arr) => arr.indexOf(v) === i); - - for (const display of candidates) { - const displayNum = display.replace(':', ''); - const socketPath = `/tmp/.X11-unix/X${displayNum}`; - - // Reuse an already-live display socket. - if (fs.existsSync(socketPath)) return display; - - try { - const xvfb = spawnFn('Xvfb', [display, '-screen', '0', '1024x768x24', '-nolisten', 'tcp'], { - detached: true, - stdio: 'ignore', - }); - xvfb.unref(); - - // Wait briefly for the X socket to become available. - execSyncFn( - `for i in 1 2 3 4 5 6 7 8 9 10; do [ -S ${socketPath} ] && exit 0; sleep 0.2; done; exit 1`, - { stdio: 'ignore' }, - ); - - return display; - } catch { - // Try next display candidate. - } - } - - return null; -}; - const checkUrlConnectivityWithBrowser = async ( url: string, browserToRun: string, @@ -559,15 +523,6 @@ const checkUrlConnectivityWithBrowser = async ( consoleLogger.info(`Unable to set download deny: ${(e as Error).message}`); } - // Give Chrome's Safe Browsing real-time service time to initialize. - // On fresh Docker containers, Chrome needs a few seconds to connect to - // Google's OHTTP relay and establish the v5 hash-prefix lookup service. - if (process.env.GOOGLE_SAFE_BROWSING) { - consoleLogger.info('[SafeBrowsing] Warming up Chrome Safe Browsing service (10s)...'); - await page.goto('about:blank'); - await new Promise(resolve => setTimeout(resolve, 10000)); - consoleLogger.info('[SafeBrowsing] Warm-up complete, navigating to target URL...'); - } // STEP 2: Navigate (follows server-side redirects) page.once('download', () => { @@ -583,29 +538,11 @@ const checkUrlConnectivityWithBrowser = async ( waitUntil: 'domcontentloaded', }); } catch (navError) { - if (process.env.GOOGLE_SAFE_BROWSING) { - const errMsg = (navError as Error).message || ''; - consoleLogger.info(`[SafeBrowsing] Navigation error: ${errMsg}`); - if (errMsg.includes('ERR_BLOCKED_BY_CLIENT') || errMsg.includes('ERR_ABORTED')) { - consoleLogger.info('[SafeBrowsing] Page blocked by Chrome Safe Browsing!'); - const currentUrl = page.url(); - consoleLogger.info(`[SafeBrowsing] Current page URL after block: ${currentUrl}`); - } - } throw navError; } if (!response) throw new Error('No response from navigation'); - if (process.env.GOOGLE_SAFE_BROWSING) { - const navUrl = page.url(); - const navStatus = response.status(); - consoleLogger.info(`[SafeBrowsing] Navigation completed - status: ${navStatus}, url: ${navUrl}`); - if (navUrl.startsWith('chrome-error:') || navUrl.includes('interstitial')) { - consoleLogger.info('[SafeBrowsing] Interstitial/error page detected!'); - } - } - // Wait briefly for JS/meta-refresh redirects to settle before reading the final URL. // Server-side redirects are already reflected after goto(), but client-side redirects // (e.g. domain.tld -> www.domain.tld via JS or meta-refresh) need extra time. @@ -1812,6 +1749,54 @@ const cloneEdgeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, des return false; }; +/** + * Clone Chrome profile Preferences file (contains Safe Browsing OHTTP key on warmed profiles) + * @param {*} options - glob options object + * @param {string} destDir - destination directory + * @returns boolean indicating whether the operation was successful + */ +const cloneChromeProfilePreferences = (options: GlobOptionsWithFileTypesFalse, destDir: string) => { + let profilePreferencesDir; + let profileNamesRegex: RegExp; + + if (os.platform() === 'win32' || os.platform() === 'darwin' || os.platform() === 'linux') { + profilePreferencesDir = globSync('**/Preferences', { + ...options, + ignore: ['oobee*/**'], + }); + } else { + return true; + } + + if (profilePreferencesDir.length > 0) { + let success = true; + profilePreferencesDir.forEach(dir => { + // Extract profile name from path (e.g., "Default" from "Default/Preferences") + const pathParts = dir.split(/[/\\]/); + const profileName = pathParts[pathParts.length - 2]; + + if (profileName && profileName !== 'oobee') { + const destProfileDir = path.join(destDir, profileName); + if (!fs.existsSync(destProfileDir)) { + fs.mkdirSync(destProfileDir, { recursive: true }); + } + + const destPrefsPath = path.join(destProfileDir, 'Preferences'); + // Always copy, overwriting if needed, to ensure OHTTP key is inherited + if (!copyFileWithRetry(dir, destPrefsPath)) { + consoleLogger.warn(`Unable to copy Chrome profile Preferences for ${profileName}.`); + success = false; + } + } + }); + return success; + } + + // If no Preferences files found in base profile, that's still OK (they'll be created) + consoleLogger.warn('[SafeBrowsing] No source Preferences files found; will be created fresh.'); + return true; +}; + /** * Both Edge and Chrome Local State files are located in the .../User Data directory * @param {*} options - glob options object @@ -1878,7 +1863,8 @@ export const cloneChromeProfiles = (randomToken: string): string => { nodir: true, }; const cloneLocalStateFileSuccess = cloneLocalStateFile(baseOptions, destDir); - if (cloneChromeProfileCookieFiles(baseOptions, destDir) && cloneLocalStateFileSuccess) { + const cloneChromePrefsSuccess = cloneChromeProfilePreferences(baseOptions, destDir); + if (cloneChromeProfileCookieFiles(baseOptions, destDir) && cloneLocalStateFileSuccess && cloneChromePrefsSuccess) { return destDir; } @@ -2287,6 +2273,69 @@ export async function launchPersistentSafeContext( return constants.launcher.launchPersistentContext(userDataDir, options); } +/** + * When GOOGLE_SAFE_BROWSING is enabled and a pre-warmed Chrome is already + * running on port 9222 (Docker), returns a Playwright launcher that connects + * via CDP using the default context (which has the active OHTTP relay). + * + * Returns null when: Safe Browsing is not enabled, browser is not Chrome, + * or no pre-warmed Chrome is reachable on port 9222. + * On macOS/Windows without a pre-warmed Chrome, callers use the standard + * launcher (Safe Browsing works natively with the cloned profile). + */ +export const getSafeBrowsingCdpLauncher = async (browser: string, _userDataDir?: string) => { + if (!process.env.GOOGLE_SAFE_BROWSING) return null; + if (browser !== 'chrome') return null; + + const { chromium } = await import('playwright'); + + try { + const testBrowser = await chromium.connectOverCDP('http://127.0.0.1:9222'); + await testBrowser.close(); + } catch { + return null; + } + + const cdpLauncher = { + launch: async () => { + consoleLogger.info('[SafeBrowsing] Connecting to pre-warmed Chrome via CDP...'); + return chromium.connectOverCDP('http://127.0.0.1:9222'); + }, + launchPersistentContext: async (_ud: string, _options: any) => { + consoleLogger.info('[SafeBrowsing] Connecting to pre-warmed Chrome via CDP...'); + const b = await chromium.connectOverCDP('http://127.0.0.1:9222'); + const ctx = b.contexts()[0]; + if (!ctx) throw new Error('[SafeBrowsing] No default context in pre-warmed Chrome'); + + const myPages = new Set(); + const origNewPage: any = ctx.newPage.bind(ctx); + ctx.newPage = async (options?: any) => { + const page = await origNewPage(options); + myPages.add(page); + page.once('close', () => myPages.delete(page)); + return page; + }; + + const origClose = ctx.close; + ctx.close = async () => { + for (const p of myPages) { + if (!p.isClosed()) await p.close().catch(() => {}); + } + myPages.clear(); + ctx.close = origClose; + ctx.newPage = origNewPage; + await b.close().catch(() => {}); + }; + + return ctx; + }, + name: () => 'chromium', + executablePath: () => '', + }; + consoleLogger.info('[SafeBrowsing] CDP launcher configured for pre-warmed Chrome on port 9222'); + return cdpLauncher; +}; + /** * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser) * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch @@ -2378,32 +2427,10 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { finalArgs.push('--enable-features=SafeBrowsingEnhancedProtection'); } - // Safe Browsing requires headful mode (Chrome doesn't enforce interstitials in headless). - // On Linux without a display, we start Xvfb to provide a virtual framebuffer. let headless = process.env.CRAWLEE_HEADLESS === '1'; - if (safeBrowsingEnabled && headless) { - if (process.platform === 'linux') { - const currentDisplay = process.env.DISPLAY; - const currentDisplayNum = (currentDisplay || '').replace(':', ''); - const currentDisplaySocket = currentDisplayNum - ? `/tmp/.X11-unix/X${currentDisplayNum}` - : ''; - - // Recover from stale DISPLAY values inherited from shell sessions. - if (!currentDisplay || !currentDisplaySocket || !fs.existsSync(currentDisplaySocket)) { - const display = ensureXvfbDisplay(); - if (display) { - process.env.DISPLAY = display; - consoleLogger.info(`[SafeBrowsing] Xvfb ready on ${display} for headful Safe Browsing`); - } else { - consoleLogger.info('[SafeBrowsing] Failed to start Xvfb, falling back to headless'); - } - } - } - if (process.env.DISPLAY) { - headless = false; - consoleLogger.info(`[SafeBrowsing] Forcing headful mode (DISPLAY=${process.env.DISPLAY})`); - } + if (safeBrowsingEnabled && headless && process.env.DISPLAY) { + headless = false; + consoleLogger.info(`[SafeBrowsing] Forcing headful mode (DISPLAY=${process.env.DISPLAY})`); } const options: LaunchOptions = { @@ -2414,37 +2441,12 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { ...(proxyOpt ? { proxy: proxyOpt } : {}), }; - // Be explicit so Chrome child processes always inherit DISPLAY when - // Safe Browsing forces headed mode in Linux containers. - if (safeBrowsingEnabled && process.platform === 'linux' && process.env.DISPLAY) { - options.env = { - ...process.env, - DISPLAY: process.env.DISPLAY, - } as NodeJS.ProcessEnv; - } - // SlowMo for debugging, can be set via env variable OOBEE_SLOWMO to avoid adding it as a CLI argument and causing confusion for users who don't need it if (!options.slowMo && process.env.OOBEE_SLOWMO && Number(process.env.OOBEE_SLOWMO) >= 1) { options.slowMo = Number(process.env.OOBEE_SLOWMO); consoleLogger.info(`Enabled browser slowMo with value: ${process.env.OOBEE_SLOWMO}ms`); } - if (safeBrowsingEnabled) { - consoleLogger.info(`[SafeBrowsing] Launch options - ignoreDefaultArgs: ${JSON.stringify(options.ignoreDefaultArgs)}`); - consoleLogger.info(`[SafeBrowsing] Launch options - headless: ${options.headless}, channel: ${channel || 'bundled-chromium'}`); - consoleLogger.info(`[SafeBrowsing] Launch options - args: ${JSON.stringify(options.args)}`); - } - - if (safeBrowsingEnabled && !!process.env.GOOGLE_SAFE_BROWSING_DEBUG) { - options.args = [ - ...(options.args ?? []), - '--enable-logging=stderr', - '--log-level=0', - '--vmodule=safe_browsing*=2,*phishing*=2', - ]; - consoleLogger.info('Safe Browsing debug logging enabled'); - } - return options; }; diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 17c0723b..c193a9f1 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -63,6 +63,16 @@ export const destinationPath = (storagePath: string): string => `${storagePath}/ */ export const getDefaultChromeDataDir = (): string => { try { + // If GOOGLE_SAFE_BROWSING is set, use the pre-warmed profile prepared by Dockerfile + if (process.env.GOOGLE_SAFE_BROWSING && fs.existsSync('/data/chrome-profile')) { + return '/data/chrome-profile'; + } + + // Check for environment override (used when GSB profile is pre-warmed in Docker) + if (process.env.OOBEE_CHROME_DATA_DIR && fs.existsSync(process.env.OOBEE_CHROME_DATA_DIR)) { + return process.env.OOBEE_CHROME_DATA_DIR; + } + let defaultChromeDataDir = null; if (os.platform() === 'win32') { defaultChromeDataDir = path.join( diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index b6623895..505b4927 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -1248,10 +1248,6 @@ export const getPreLaunchHook = (userDataDirectory: string) => { await fsp.mkdir(effectiveDir, { recursive: true }); - if (process.env.GOOGLE_SAFE_BROWSING) { - injectSafeBrowsingDb(effectiveDir); - } - // Copy auth-relevant files from the pristine base directory so // authenticated sessions are preserved across pool rotations. try { @@ -1270,6 +1266,14 @@ export const getPreLaunchHook = (userDataDirectory: string) => { const destProfile = path.join(effectiveDir, profile.name); await fsp.mkdir(destProfile, { recursive: true }).catch(() => {}); + // Copy Preferences so the Safe Browsing OHTTP key (hash_real_time_ohttp_key) + // is inherited from the base profile. injectSafeBrowsingDb() will merge + // enabled/enhanced on top, preserving the warmed-up key. + const prefsSrc = path.join(srcProfile, 'Preferences'); + if (await fsp.stat(prefsSrc).catch(() => null)) { + await fsp.copyFile(prefsSrc, path.join(destProfile, 'Preferences')).catch(() => {}); + } + // Cookies (macOS layout: /Cookies) const cookiesSrc = path.join(srcProfile, 'Cookies'); if (await fsp.stat(cookiesSrc).catch(() => null)) { @@ -1305,6 +1309,12 @@ export const getPreLaunchHook = (userDataDirectory: string) => { // Silent fallback: use empty profile if clone fails } + // Inject Safe Browsing preferences after copying base profile files. + // This merges enabled/enhanced on top of any copied Preferences, preserving + // the hash_real_time_ohttp_key from the base profile so Chrome does not need + // to re-fetch it for every pool browser. + injectSafeBrowsingDb(effectiveDir); + // Clean any stale lock files that may block browser launches on Windows const lockFiles = [ path.join(effectiveDir, 'SingletonLock'), @@ -1348,6 +1358,7 @@ export const getPostPageCloseHook = (userDataDirectory: string) => { }; }; + export const failedRequestHandler = async ({ request }: { request: Request }) => { guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url }); log.error(`Failed Request - ${request.url}: ${request.errorMessages}`); diff --git a/src/crawlers/crawlDomain.ts b/src/crawlers/crawlDomain.ts index 80bf41d6..865e8376 100644 --- a/src/crawlers/crawlDomain.ts +++ b/src/crawlers/crawlDomain.ts @@ -26,6 +26,7 @@ import constants, { } from '../constants/constants.js'; import { getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, @@ -394,19 +395,21 @@ const crawlDomain = async ({ const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders); + const cdpLauncher = await getSafeBrowsingCdpLauncher(browser, userDataDirectory); + const crawler = register( new crawlee.PlaywrightCrawler({ launchContext: { - launcher: constants.launcher, + launcher: (cdpLauncher || constants.launcher) as any, launchOptions: getPlaywrightLaunchOptions(browser), }, retryOnBlocked: false, browserPoolOptions: { useFingerprints: false, - retireBrowserAfterPageCount: 500, + retireBrowserAfterPageCount: cdpLauncher ? Number.MAX_SAFE_INTEGER : 500, closeInactiveBrowserAfterSecs: 30, preLaunchHooks: [ - getPreLaunchHook(userDataDirectory), + ...(!cdpLauncher ? [getPreLaunchHook(userDataDirectory)] : []), async (_pageId, launchContext) => { // eslint-disable-next-line no-param-reassign launchContext.launchOptions = { @@ -420,7 +423,7 @@ const crawlDomain = async ({ }; }, ], - postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)], + postPageCloseHooks: [...(!cdpLauncher ? [getPostPageCloseHook(userDataDirectory)] : [])], }, requestQueue, maxRequestRetries: 3, @@ -523,6 +526,7 @@ const crawlDomain = async ({ } if (actualUrl.startsWith('chrome-error:')) { + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING; guiInfoLog(guiInfoStatusTypes.SKIPPED, { numScanned: urlsCrawled.scanned.length, urlScanned: request.url, @@ -531,10 +535,9 @@ const crawlDomain = async ({ url: request.url, pageTitle: request.url, actualUrl: request.url, - metadata: STATUS_CODE_METADATA[3], - httpStatusCode: 3, + metadata: isSafeBrowsingBlock ? STATUS_CODE_METADATA[3] : STATUS_CODE_METADATA[1], + httpStatusCode: isSafeBrowsingBlock ? 3 : 1, }); - await enqueueProcess(page, enqueueLinks, browserContext); return; } @@ -897,6 +900,27 @@ const crawlDomain = async ({ return; } + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING && + request.errorMessages?.some((msg: string) => + msg.includes('ERR_BLOCKED_BY_CLIENT') || + msg.includes('ERR_BLOCKED_BY_RESPONSE'), + ); + + if (isSafeBrowsingBlock) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: urlsCrawled.scanned.length, urlScanned: request.url, diff --git a/src/crawlers/crawlSitemap.ts b/src/crawlers/crawlSitemap.ts index 3c6916d7..6374800b 100644 --- a/src/crawlers/crawlSitemap.ts +++ b/src/crawlers/crawlSitemap.ts @@ -24,6 +24,7 @@ import constants, { import { getLinksFromSitemap, getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, isDisallowedInRobotsTxt, isSkippedUrl, waitForPageLoaded, @@ -144,19 +145,21 @@ const crawlSitemap = async ({ // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery. const { requestQueue } = await createCrawleeSubFolders(randomToken); + const cdpLauncher = await getSafeBrowsingCdpLauncher(browser, userDataDirectory); + const crawler = register( new crawlee.PlaywrightCrawler({ launchContext: { - launcher: constants.launcher, + launcher: (cdpLauncher || constants.launcher) as any, launchOptions: getPlaywrightLaunchOptions(browser), }, retryOnBlocked: false, browserPoolOptions: { useFingerprints: false, - retireBrowserAfterPageCount: 500, + retireBrowserAfterPageCount: cdpLauncher ? Number.MAX_SAFE_INTEGER : 500, closeInactiveBrowserAfterSecs: 30, preLaunchHooks: [ - getPreLaunchHook(userDataDirectory), + ...(!cdpLauncher ? [getPreLaunchHook(userDataDirectory)] : []), async (_pageId, launchContext) => { launchContext.launchOptions = { ...launchContext.launchOptions, @@ -169,7 +172,7 @@ const crawlSitemap = async ({ }; }, ], - postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)], + postPageCloseHooks: [...(!cdpLauncher ? [getPostPageCloseHook(userDataDirectory)] : [])], }, requestList, requestQueue, @@ -589,6 +592,27 @@ const crawlSitemap = async ({ return; } + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING && + request.errorMessages?.some((msg: string) => + msg.includes('ERR_BLOCKED_BY_CLIENT') || + msg.includes('ERR_BLOCKED_BY_RESPONSE'), + ); + + if (isSafeBrowsingBlock) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: urlsCrawled.scanned.length, urlScanned: request.url, diff --git a/src/crawlers/custom/utils.ts b/src/crawlers/custom/utils.ts index b98cc735..4a9d7bb1 100644 --- a/src/crawlers/custom/utils.ts +++ b/src/crawlers/custom/utils.ts @@ -310,7 +310,12 @@ export const addOverlayMenu = async ( collapsed: false, }, ) => { - await page.waitForLoadState('domcontentloaded', { timeout: OVERLAY_OPERATION_TIMEOUT_MS }); + try { + await page.waitForLoadState('domcontentloaded', { timeout: 2000 }); + } catch { + // In CDP mode the load state may not resolve after script injection (e.g. axe-core). + // Proceed with injection anyway — the DOM is accessible if evaluate() succeeds. + } consoleLogger.info(`Overlay menu: adding to ${menuPos}...`); // Add the overlay menu with initial styling @@ -1211,11 +1216,19 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa .then(async () => { if (refreshSeq !== overlayRefreshSeq || page.isClosed()) return; + // During an active scan, navigation events (framenavigated/domcontentloaded) can fire + // due to axe-core injection or page resource loading. In CDP mode, concurrent + // page.evaluate() calls conflict with the running scan. Skip overlay injection + // for non-scan triggers while scanning — the overlay will be re-added by the + // 'scan-click' trigger after the scan completes. + if (pagesDict[pageId]?.isScanning && trigger !== 'scan-click') return; + try { // `framenavigated` can fire before the new document is ready for DOM inspection/injection. - await page.waitForLoadState('domcontentloaded', { timeout: 5000 }); + // Use a short timeout — in CDP mode, waitForLoadState can hang after script injection. + await page.waitForLoadState('domcontentloaded', { timeout: 2000 }); } catch { - // Best effort only. The page may still be mid-navigation. + // Best effort only. The page may still be mid-navigation or state tracking confused. } try { @@ -1306,6 +1319,18 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa if (page.isClosed()) return; await reconcileOverlayMenu('scan-click'); + + // If the overlay still isn't present after the first attempt (can happen in + // CDP mode where waitForLoadState tracking is unreliable), retry once. + if (!page.isClosed()) { + const overlayPresent = await page.evaluate(() => + Boolean(document.querySelector('#oobeeShadowHost')), + ).catch(() => false); + if (!overlayPresent) { + log('Overlay missing after scan-click reconcile, retrying...'); + await reconcileOverlayMenu('scan-click'); + } + } } catch (error) { log(`Scan failed ${error}`); } diff --git a/src/crawlers/runCustom.ts b/src/crawlers/runCustom.ts index a2e03876..c4dd1e06 100644 --- a/src/crawlers/runCustom.ts +++ b/src/crawlers/runCustom.ts @@ -7,12 +7,13 @@ import constants, { UrlsCrawled, } from '../constants/constants.js'; import { DEBUG, initNewPage, log } from './custom/utils.js'; -import { guiInfoLog } from '../logs.js'; +import { consoleLogger, guiInfoLog } from '../logs.js'; import { ViewportSettingsClass } from '../combine.js'; import { addUrlGuardScript } from './guards/urlGuard.js'; import { getBrowserToRun, getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, initModifiedUserAgent, launchPersistentSafeContext, } from '../constants/common.js'; @@ -113,18 +114,33 @@ const runCustom = async ( const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders); - const context = await launchPersistentSafeContext(userDataDirectory, { - ...baseLaunchOptions, - args: mergedArgs, - headless: false, - ignoreHTTPSErrors: true, - serviceWorkers: 'block', - viewport: null, - ...(hasCustomViewport ? contextDeviceOptions : {}), - userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined), - ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), - ...(httpCredentials && { httpCredentials }), - }); + const cdpLauncher = await getSafeBrowsingCdpLauncher(resolvedBrowserToRun, userDataDirectory); + let context; + let usedCdp = false; + + if (cdpLauncher) { + try { + context = await cdpLauncher.launchPersistentContext(userDataDirectory, {}); + usedCdp = true; + } catch (e) { + consoleLogger.info(`[SafeBrowsing] CDP fallback: ${e}`); + } + } + + if (!context) { + context = await launchPersistentSafeContext(userDataDirectory, { + ...baseLaunchOptions, + args: mergedArgs, + headless: false, + ignoreHTTPSErrors: true, + serviceWorkers: 'block' as const, + viewport: null, + ...(hasCustomViewport ? contextDeviceOptions : {}), + userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined), + ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), + ...(httpCredentials && { httpCredentials }), + }); + } if (authHeader) { await addAuthRouteHandler(context, url, authHeader); @@ -143,7 +159,9 @@ const runCustom = async ( addUrlGuardScript(context, { fallbackUrl: url, allowChromeErrors: !!process.env.GOOGLE_SAFE_BROWSING }); - const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage()); + const page = usedCdp + ? await context.newPage() + : context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage()); await initNewPage(page, pageClosePromises, processPageParams, pagesDict); // Detection of new page