diff --git a/AGENTS.md b/AGENTS.md index ec5ea56b..f30e25df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,8 +144,10 @@ The `constants` default export object holds runtime state: | `OOBEE_SCAN_PRODUCT` | Adds `scanProduct` tag to Sentry events | | `OOBEE_CONSECUTIVE_MAX_RETRIES` | Max consecutive HTTP failures before circuit breaker aborts crawl (default 100) | | `OOBEE_VALIDATE_URL` | If set, exit after URL validation without scanning | +<<<<<<< HEAD | `OOBEE_SAVE_DOM` | `1` or `true` = save full-page DOM HTML to `pageDOMs/` in results directory. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | | `OOBEE_SAVE_PAGE_SCREENSHOT` | `1` or `true` = save full-page desktop + mobile viewport screenshots to `pageDOMs/desktopPageScreenshots/` and `pageDOMs/mobilePageScreenshots/`. Mobile viewport uses iPhone 11 width programmatically. Supported scan types: Website, Sitemap, Intelligent, LocalFile, Custom | +| `GOOGLE_SAFE_BROWSING` | `1` = enable Google Safe Browsing (requires Chrome, not Chromium) | | `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` | Proxy configuration | | `NO_PROXY` / `INCLUDE_PROXY` | Proxy bypass/include lists | @@ -180,6 +182,71 @@ The `constants` default export object holds runtime state: - Path separator differences in cookie profile regex - `CRAWLEE_SYSTEM_INFO_V2=1` needed (wmic deprecation) +## Safe Browsing + +### Overview + +Google Safe Browsing protects users by blocking navigation to phishing/malware URLs. On macOS, it copies the local hash-prefix threat database (`UrlSoceng.store.*`, `UrlMalware.store.*`) from the system Chrome profile. On Docker Linux, it connects via CDP to a pre-warmed Chrome instance with an active OHTTP relay. Blocked pages are classified as "Blocked by Safe Browsing" in scan results. + +### Requirements + +1. **Google Chrome** (not Chromium) — Safe Browsing requires Google's proprietary API keys. +2. **`GOOGLE_SAFE_BROWSING`** environment variable set to any value. +3. **macOS or Linux only** — Windows is not yet supported (prints a warning and skips). + +### How It Works + +#### macOS (all scan types) + +1. `ensureAndInjectSafeBrowsing()` copies the Safe Browsing DB from your system Chrome profile (`~/Library/Application Support/Google/Chrome/Safe Browsing/`) into a base profile at `~/.oobee/safe-browsing-profile/`. +2. `injectSafeBrowsingDb()` copies the DB into each scan's browser profile directory and writes `safebrowsing: { enabled: true, enhanced: true }` into Preferences. +3. `getPlaywrightLaunchOptions()` removes Playwright default flags that suppress Safe Browsing (`--safebrowsing-disable-auto-update`, `--disable-client-side-phishing-detection`, `--disable-background-networking`, `--disable-component-update`) via `ignoreDefaultArgs`. +4. Chrome uses the local DB for immediate threat detection without needing the OHTTP relay. + +#### Docker Linux (crawlDomain, crawlSitemap) + +1. A pre-warmed Chrome runs on port 9222 (started by `start-gsb-novnc.sh`) with an active OHTTP relay. +2. `getSafeBrowsingCdpLauncher()` connects to this Chrome via CDP and returns the default context (which has active Safe Browsing). +3. Crawlee's browser pool uses this CDP launcher instead of spawning fresh instances. + +#### Docker Linux (runCustom) + +1. Same CDP approach — `getSafeBrowsingCdpLauncher()` connects to the pre-warmed Chrome. +2. Falls back to `launchPersistentSafeContext()` if CDP is unavailable. + +#### Detection of blocked pages + +- **requestHandler**: Pages that navigate to `chrome-error://` are classified as "Blocked by Safe Browsing" (`STATUS_CODE_METADATA[3]`). +- **failedRequestHandler**: Navigation errors containing `ERR_BLOCKED_BY_CLIENT` or `ERR_BLOCKED_BY_RESPONSE` are classified as "Blocked by Safe Browsing". +- **runCustom**: `urlGuard.ts` with `allowChromeErrors: true` lets the interstitial page stay visible to the user. + +### Docker / Architecture Constraints + +- **amd64 (x86_64)**: Google Chrome .deb is installed in Dockerfile. Safe Browsing works fully. +- **arm64 (aarch64)**: Chrome is NOT available for ARM64 Linux. Safe Browsing is unavailable. +- Build the image with `docker build --platform linux/amd64` to ensure Chrome is available. + +### Environment Variables + +| Variable | Purpose | +|----------|---------| +| `GOOGLE_SAFE_BROWSING` | Enable Safe Browsing (any value; requires Chrome) | + +### Key Files + +- `src/safeBrowsingProfile.ts` — DB warmup, copy from system profile, injection into scan profiles +- `src/constants/common.ts` — `getSafeBrowsingCdpLauncher()`, `getPlaywrightLaunchOptions()` (ignoreDefaultArgs), `launchPersistentSafeContext()` +- `src/crawlers/guards/urlGuard.ts` — `allowChromeErrors` for interstitial pages in runCustom +- `src/crawlers/crawlDomain.ts` — `chrome-error:` and `ERR_BLOCKED_BY_CLIENT` detection +- `src/crawlers/crawlSitemap.ts` — same blocked-page detection +- `src/crawlers/runCustom.ts` — CDP on Docker, persistent context on macOS + +### What Does NOT Work + +- Chromium (Playwright's bundled browser) — lacks Safe Browsing entirely +- Windows — not yet supported (prints warning) +- ARM64 Linux Docker — Chrome .deb not published for arm64 + ## Testing ```bash diff --git a/Dockerfile b/Dockerfile index 41d98695..a737219f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,58 +2,77 @@ # Node version is v22 FROM mcr.microsoft.com/playwright:v1.61.1-noble + # Installation of packages for oobee RUN apt-get update && apt-get install -y --no-install-recommends \ git \ unzip \ - zip && \ + zip \ + xvfb \ + x11vnc \ + novnc \ + websockify \ + openbox \ + procps \ + libnss3-tools && \ rm -rf /var/lib/apt/lists/* - -WORKDIR /app/oobee -# Clone oobee repository -# RUN git clone --branch master https://github.com/GovTechSG/oobee.git /app/oobee +# Install Playwright browsers +RUN npx playwright install chromium -# OR Copy oobee files from local directory -COPY . . +# ============================================================================= +# Google Chrome installation for Safe Browsing support +# ============================================================================= +# WHY: Chrome's Safe Browsing (v5) checks URLs in real-time via Google's OHTTP +# relay — no local hash database is needed or pre-seeded. +# This is Chrome-only; Chromium lacks the required proprietary API keys. +# +# ARCHITECTURE LIMITATION: +# Chrome .deb packages are only available for amd64 (x86_64). +# On arm64 builds this step is skipped and Safe Browsing will not be available. +# +# TO ENABLE: Set env var GOOGLE_SAFE_BROWSING=1 when running the container. +# ============================================================================= +RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ + wget -q -O /tmp/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get update && apt-get install -y --no-install-recommends /tmp/chrome.deb && \ + rm -f /tmp/chrome.deb && rm -rf /var/lib/apt/lists/*; \ + else \ + echo "NOTICE: Skipping Chrome install (Safe Browsing unavailable on $(dpkg --print-architecture))"; \ + fi + + +# --- noVNC / Safe Browsing warmup entrypoint --- +# Baked into the base image so the same oobee image can be used as a GUI +# test container by overriding the entrypoint: +# docker run ... -p 6080:6080 oobee /usr/local/bin/start-gsb-novnc.sh +COPY gsb-test/start-gsb-novnc.sh /usr/local/bin/start-gsb-novnc.sh +RUN chmod +x /usr/local/bin/start-gsb-novnc.sh +ENV GSB_WARMUP_SECONDS=300 +ENV GSB_WARMUP_URL=https://example.com +ENV GOOGLE_SAFE_BROWSING=1 +EXPOSE 6080 5900 +VOLUME ["/data/chrome-profile"] + +# --- App code (changes here don't invalidate Chrome layers above) --- # Environment variables for node and Playwright ENV NODE_ENV=production ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="true" -# Install oobee dependencies -# TODO: Move back to npm ci --omit=dev once module package-lock issue vs MacOS is resolved -RUN npm install --omit=dev - -# Compile TypeScript for oobee -RUN npm run build || true # true exits with code 0 - workaround for TS errors - -# Install Playwright browsers -RUN npx playwright install chromium - -# Add non-privileged user -# Create a group named "purple" -RUN groupadd -r purple +# Add non-privileged user before app copy so ownership can be set during COPY. +RUN groupadd -r purple && useradd -r -g purple purple && \ + mkdir -p /home/purple /app/oobee && chown purple:purple /home/purple /app /app/oobee -# Create a user named "purple" and assign it to the group "purple" -RUN useradd -r -g purple purple - -# Create a dedicated directory for the "purple" user and set permissions -RUN mkdir -p /home/purple && chown -R purple:purple /home/purple - -WORKDIR /app - -# Set the ownership of the oobee directory to the user "purple" -RUN chown -R purple:purple /app - -# Copy any application and support files -# COPY . . - -# Install any app dependencies for your application -# RUN npm ci --omit=dev - -# For oobee to be run from present working directory, comment out as necessary WORKDIR /app/oobee -# Run everything after as non-privileged user. +# Run app build steps as non-root to avoid a full recursive chown later. USER purple + +# Install dependencies first (cached unless package.json/package-lock.json change) +COPY --chown=purple:purple package.json package-lock.json ./ +RUN npm install --omit=dev + +# Copy source and compile TypeScript +COPY --chown=purple:purple . . +RUN npm run build || true # true exits with code 0 - workaround for TS errors diff --git a/README.md b/README.md index 4dec73c4..811e7d84 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ verapdf --version | ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | | | NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | | | INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | | +| GOOGLE_SAFE_BROWSING | When set, enables Google Safe Browsing URL protection. Blocks phishing, malware, and unwanted software URLs — blocked pages are classified as "Blocked by Safe Browsing" in reports. Requires Google Chrome (not Chromium). On macOS, copies the threat database from your system Chrome profile. On Docker Linux, connects to a pre-warmed Chrome instance. macOS and Linux only (Windows not yet supported). | | #### Environment variables used internally (Do not set) Do not set these environment variables or behaviour might change unexpectedly. diff --git a/gsb-test/start-gsb-novnc.sh b/gsb-test/start-gsb-novnc.sh new file mode 100644 index 00000000..87aef96a --- /dev/null +++ b/gsb-test/start-gsb-novnc.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +export DISPLAY=${DISPLAY:-:99} +PROFILE_DIR=${PROFILE_DIR:-/data/chrome-profile} +export OOBEE_CHROME_DATA_DIR="$PROFILE_DIR" +WARMUP_SECONDS=${GSB_WARMUP_SECONDS:-300} +CHROME_URL=${GSB_WARMUP_URL:-https://example.com} + +mkdir -p "$PROFILE_DIR/Default" + +# Import Cloudflare CA into Chrome's NSS database so HTTPS interception +# by corporate proxies doesn't trigger "connection is not private" errors. +NSS_DB="$PROFILE_DIR/Default" +if [ -f /usr/local/share/ca-certificates/Cloudflare_CA.crt ] && command -v certutil >/dev/null 2>&1; then + mkdir -p "$NSS_DB" + if [ ! -d "$NSS_DB" ] || ! certutil -L -d sql:"$NSS_DB" -n "Cloudflare CA" >/dev/null 2>&1; then + certutil -d sql:"$NSS_DB" -N --empty-password 2>/dev/null || true + certutil -d sql:"$NSS_DB" -A -t "C,," -n "Cloudflare CA" \ + -i /usr/local/share/ca-certificates/Cloudflare_CA.crt + echo "[GSB] Cloudflare CA imported into Chrome NSS database" + fi +fi + +# Ensure Safe Browsing is enabled in profile Preferences on first run. +if [ ! -f "$PROFILE_DIR/Default/Preferences" ]; then + echo '{"safebrowsing":{"enabled":true,"enhanced":true}}' > "$PROFILE_DIR/Default/Preferences" +fi + +rm -f /tmp/.X99-lock +Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp >/tmp/xvfb.log 2>&1 & +openbox >/tmp/openbox.log 2>&1 & +x11vnc -display :99 -forever -shared -nopw -listen 0.0.0.0 -rfbport 5900 >/tmp/x11vnc.log 2>&1 & + +# Start Chrome and keep it alive in the background for the lifetime of the container. +# This keeps the Safe Browsing real-time OHTTP relay connection warm so the first +# Oobee scan does not have to wait for the ~5 minute cold-start initialization. +CHROME_ARGS=( + --user-data-dir="$PROFILE_DIR" + --profile-directory=Default + --no-sandbox --disable-setuid-sandbox + --disable-dev-shm-usage --disable-gpu --disable-software-rasterizer + --in-process-gpu --disable-gpu-compositing --disable-features=VizDisplayCompositor + --no-zygote --no-first-run --no-default-browser-check + --enable-features=SafeBrowsingEnhancedProtection + --ignore-certificate-errors + --ozone-platform=x11 + --remote-debugging-port=9222 +) + +if command -v google-chrome >/dev/null 2>&1; then + rm -f "$PROFILE_DIR/SingletonLock" "$PROFILE_DIR/SingletonCookie" "$PROFILE_DIR/SingletonSocket" + google-chrome "${CHROME_ARGS[@]}" "$CHROME_URL" >/tmp/chrome-warmup.log 2>&1 & + CHROME_PID=$! + echo "[GSB] Chrome started (PID $CHROME_PID) — waiting for OHTTP key (max ${WARMUP_SECONDS}s)..." + + WAITED=0 + ATTEMPT=0 + GSB_READY=0 + MAX_ATTEMPTS=$(( WARMUP_SECONDS / 5 )) + while [ "$WAITED" -lt "$WARMUP_SECONDS" ]; do + sleep 5 + WAITED=$((WAITED + 5)) + ATTEMPT=$((ATTEMPT + 1)) + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — checking for OHTTP key (${WAITED}s elapsed)..." + KEY=$(python3 -c " +import json, sys +try: + with open('$PROFILE_DIR/Default/Preferences') as f: + d = json.load(f) + print(d.get('safebrowsing', {}).get('hash_real_time_ohttp_key', '')) +except Exception: + print('') +" 2>/dev/null || true) + if [ -n "$KEY" ]; then + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — OHTTP key present after ${WAITED}s — Safe Browsing READY" + GSB_READY=1 + break + fi + echo "[GSB] Attempt ${ATTEMPT}/${MAX_ATTEMPTS} — key not yet present, will retry in 5s" + done + + if [ "$GSB_READY" -eq 0 ]; then + echo "[GSB] WARNING: OHTTP key not found after ${WARMUP_SECONDS}s — Safe Browsing may not be fully ready" + fi + echo "[GSB] Chrome remains running (PID $CHROME_PID)" +fi + +if [ -x /usr/share/novnc/utils/novnc_proxy ]; then + exec /usr/share/novnc/utils/novnc_proxy --listen 6080 --vnc localhost:5900 +else + exec websockify --web=/usr/share/novnc 6080 localhost:5900 +fi diff --git a/src/constants/common.ts b/src/constants/common.ts index 20c95aac..b8c76d70 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -14,6 +14,7 @@ import url, { fileURLToPath, pathToFileURL } from 'url'; import safe from 'safe-regex'; import * as https from 'https'; import os from 'os'; + import mime from 'mime'; import { minimatch } from 'minimatch'; import { globSync, GlobOptionsWithFileTypesFalse } from 'glob'; @@ -37,6 +38,7 @@ import { cleanUpAndExit, isFollowStrategy, randomThreeDigitNumberString, registe import { Answers, Data } from '../index.js'; import { DeviceDescriptor } from '../types/types.js'; import { getProxyInfo, proxyInfoToResolution, ProxySettings } from '../proxyService.js'; +import { ensureAndInjectSafeBrowsing } from '../safeBrowsingProfile.js'; // validateDirPath validates a provided directory path // returns null if no error @@ -307,6 +309,30 @@ const isAllowedContentType = (ct: string): boolean => { ); }; +const isLikelyQemuChromeCrash = (error: unknown): boolean => { + const msg = error instanceof Error ? error.message : String(error || ''); + return ( + msg.includes('Target page, context or browser has been closed') || + msg.includes("GPU process isn't usable") || + msg.includes('qemu: uncaught target signal') + ); +}; + +const withQemuChromeWorkaroundArgs = (launchOptions: LaunchOptions): LaunchOptions => { + const args = [...(launchOptions.args ?? [])]; + const extraArgs = [ + '--in-process-gpu', + '--disable-gpu-compositing', + '--disable-features=VizDisplayCompositor', + ]; + + extraArgs.forEach(arg => { + if (!args.includes(arg)) args.push(arg); + }); + + return { ...launchOptions, args }; +}; + const checkUrlConnectivityWithBrowser = async ( url: string, browserToRun: string, @@ -402,32 +428,63 @@ const checkUrlConnectivityWithBrowser = async ( // Keep UA emulation explicitly. contextOptions.userAgent = process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined); - try { - const launchPersistent = async () => { - browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, { - ...launchOptions, - ...contextOptions, - }); - register(browserContext); - }; + const launchPersistent = async (effectiveLaunchOptions: LaunchOptions) => { + browserContext = await launchPersistentSafeContext(clonedDataDir, { + ...effectiveLaunchOptions, + ...contextOptions, + }); + register(browserContext); + }; - const launchEphemeral = async () => { - browserInstance = await constants.launcher.launch(launchOptions); - register(browserInstance as unknown as { close: () => Promise }); - browserContext = await browserInstance.newContext(contextOptions); - }; + const launchEphemeral = async (effectiveLaunchOptions: LaunchOptions) => { + browserInstance = await constants.launcher.launch(effectiveLaunchOptions); + register(browserInstance as unknown as { close: () => Promise }); + browserContext = await browserInstance.newContext(contextOptions); + }; + const launchBrowserContext = async (effectiveLaunchOptions: LaunchOptions) => { if (process.env.CRAWLEE_HEADLESS === '1') { try { - await launchPersistent(); + await launchPersistent(effectiveLaunchOptions); } catch (error) { // Fallback to ephemeral context if persistent context fails (e.g. protocol errors) // More prone to falling back here when running localFile scans consoleLogger.warn(`Persistent context launch failed, retrying with ephemeral context: ${error.message}`); - await launchEphemeral(); + await launchEphemeral(effectiveLaunchOptions); } } else { - await launchEphemeral(); + await launchEphemeral(effectiveLaunchOptions); + } + }; + + try { + try { + await launchBrowserContext(launchOptions); + } catch (error) { + const shouldRetryWithQemuArgs = + browserToRun === BrowserTypes.CHROME && + process.platform === 'linux' && + fs.existsSync('/.dockerenv') && + isLikelyQemuChromeCrash(error); + + if (!shouldRetryWithQemuArgs) throw error; + + consoleLogger.warn( + '[Chrome] Launch failed with likely emulation/GPU issue. Retrying Chrome with QEMU-safe flags.', + ); + + try { + await browserContext?.close(); + } catch {} + if (browserInstance) { + try { + await browserInstance.close(); + } catch {} + } + browserContext = undefined; + browserInstance = undefined; + + await launchBrowserContext(withQemuChromeWorkaroundArgs(launchOptions)); } } catch (err) { printMessage([`Unable to launch browser\n${err}`], messageOptions); @@ -466,17 +523,23 @@ const checkUrlConnectivityWithBrowser = async ( consoleLogger.info(`Unable to set download deny: ${(e as Error).message}`); } + // STEP 2: Navigate (follows server-side redirects) page.once('download', () => { res.status = constants.urlCheckStatuses.notASupportedDocument.code; return res; }); - + // OPTIMIZATION: Wait for 'domcontentloaded' only - const response = await page.goto(url, { - timeout: 15000, - waitUntil: 'domcontentloaded', // enough to get status + allow potential client redirects to kick in - }); + let response; + try { + response = await page.goto(url, { + timeout: 15000, + waitUntil: 'domcontentloaded', + }); + } catch (navError) { + throw navError; + } if (!response) throw new Error('No response from navigation'); @@ -929,7 +992,7 @@ const getRobotsTxtViaPlaywright = async ( try { if (process.env.CRAWLEE_HEADLESS === '1') { - browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, { + browserContext = await launchPersistentSafeContext(robotsDataDir, { ...getPlaywrightLaunchOptions(browser), ...(extraHTTPHeaders && { extraHTTPHeaders }), ...(process.env.OOBEE_USER_AGENT && { userAgent: process.env.OOBEE_USER_AGENT }), @@ -1151,7 +1214,7 @@ export const getLinksFromSitemap = async ( try { if (process.env.CRAWLEE_HEADLESS === '1') { - browserContext = await constants.launcher.launchPersistentContext( + browserContext = await launchPersistentSafeContext( finalUserDataDirectory, { ...getPlaywrightLaunchOptions(browser), @@ -1592,6 +1655,8 @@ const cloneChromeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, d ignore: 'oobee*/**', }); profileNamesRegex = /Chrome\/(.*?)\/Cookies/; + } else { + return true; } if (profileCookiesDir.length > 0) { @@ -1684,6 +1749,54 @@ const cloneEdgeProfileCookieFiles = (options: GlobOptionsWithFileTypesFalse, des return false; }; +/** + * Clone Chrome profile Preferences file (contains Safe Browsing OHTTP key on warmed profiles) + * @param {*} options - glob options object + * @param {string} destDir - destination directory + * @returns boolean indicating whether the operation was successful + */ +const cloneChromeProfilePreferences = (options: GlobOptionsWithFileTypesFalse, destDir: string) => { + let profilePreferencesDir; + let profileNamesRegex: RegExp; + + if (os.platform() === 'win32' || os.platform() === 'darwin' || os.platform() === 'linux') { + profilePreferencesDir = globSync('**/Preferences', { + ...options, + ignore: ['oobee*/**'], + }); + } else { + return true; + } + + if (profilePreferencesDir.length > 0) { + let success = true; + profilePreferencesDir.forEach(dir => { + // Extract profile name from path (e.g., "Default" from "Default/Preferences") + const pathParts = dir.split(/[/\\]/); + const profileName = pathParts[pathParts.length - 2]; + + if (profileName && profileName !== 'oobee') { + const destProfileDir = path.join(destDir, profileName); + if (!fs.existsSync(destProfileDir)) { + fs.mkdirSync(destProfileDir, { recursive: true }); + } + + const destPrefsPath = path.join(destProfileDir, 'Preferences'); + // Always copy, overwriting if needed, to ensure OHTTP key is inherited + if (!copyFileWithRetry(dir, destPrefsPath)) { + consoleLogger.warn(`Unable to copy Chrome profile Preferences for ${profileName}.`); + success = false; + } + } + }); + return success; + } + + // If no Preferences files found in base profile, that's still OK (they'll be created) + consoleLogger.warn('[SafeBrowsing] No source Preferences files found; will be created fresh.'); + return true; +}; + /** * Both Edge and Chrome Local State files are located in the .../User Data directory * @param {*} options - glob options object @@ -1750,7 +1863,8 @@ export const cloneChromeProfiles = (randomToken: string): string => { nodir: true, }; const cloneLocalStateFileSuccess = cloneLocalStateFile(baseOptions, destDir); - if (cloneChromeProfileCookieFiles(baseOptions, destDir) && cloneLocalStateFileSuccess) { + const cloneChromePrefsSuccess = cloneChromeProfilePreferences(baseOptions, destDir); + if (cloneChromeProfileCookieFiles(baseOptions, destDir) && cloneLocalStateFileSuccess && cloneChromePrefsSuccess) { return destDir; } @@ -2034,7 +2148,7 @@ export const submitFormViaPlaywright = async ( userDataDirectory: string, finalUrl: string, ) => { - const browserContext = await constants.launcher.launchPersistentContext(userDataDirectory, { + const browserContext = await launchPersistentSafeContext(userDataDirectory, { ...getPlaywrightLaunchOptions(browserToRun), }); @@ -2114,11 +2228,12 @@ export async function initModifiedUserAgent( ) { // UA bootstrap must not use persistent context / user-data-dir. const launchOptions = getPlaywrightLaunchOptions(browser); - - const browserInstance = await constants.launcher.launch(launchOptions); - register(browserInstance as unknown as { close: () => Promise }); + let browserInstance: Awaited> | undefined; try { + browserInstance = await constants.launcher.launch(launchOptions); + register(browserInstance as unknown as { close: () => Promise }); + const context = await browserInstance.newContext(); const page = await context.newPage(); const defaultUA = await page.evaluate(() => navigator.userAgent); @@ -2130,19 +2245,105 @@ export async function initModifiedUserAgent( // Do not mutate global CLI args with --user-agent= process.env.OOBEE_USER_AGENT = modifiedUA; + } catch (error) { + const fallbackUA = + (typeof (_playwrightDeviceDetailsObject as any)?.userAgent === 'string' && + (_playwrightDeviceDetailsObject as any).userAgent) || + devices['Desktop Chrome'].userAgent; + + process.env.OOBEE_USER_AGENT = fallbackUA.includes('HeadlessChrome') + ? fallbackUA.replace('HeadlessChrome', 'Chrome') + : fallbackUA; + + consoleLogger.warn( + `[UA] Failed to bootstrap user agent from browser (${(error as Error).message}). Using fallback Chrome UA string.`, + ); } finally { - await browserInstance.close(); + await browserInstance?.close(); } } const cacheProxyInfo = getProxyInfo(); +export async function launchPersistentSafeContext( + userDataDir: string, + options: Parameters[1], +) { + await ensureAndInjectSafeBrowsing(userDataDir); + return constants.launcher.launchPersistentContext(userDataDir, options); +} + +/** + * When GOOGLE_SAFE_BROWSING is enabled and a pre-warmed Chrome is already + * running on port 9222 (Docker), returns a Playwright launcher that connects + * via CDP using the default context (which has the active OHTTP relay). + * + * Returns null when: Safe Browsing is not enabled, browser is not Chrome, + * or no pre-warmed Chrome is reachable on port 9222. + * On macOS/Windows without a pre-warmed Chrome, callers use the standard + * launcher (Safe Browsing works natively with the cloned profile). + */ +export const getSafeBrowsingCdpLauncher = async (browser: string, _userDataDir?: string) => { + if (!process.env.GOOGLE_SAFE_BROWSING) return null; + if (browser !== 'chrome') return null; + + const { chromium } = await import('playwright'); + + try { + const testBrowser = await chromium.connectOverCDP('http://127.0.0.1:9222'); + await testBrowser.close(); + } catch { + return null; + } + + const cdpLauncher = { + launch: async () => { + consoleLogger.info('[SafeBrowsing] Connecting to pre-warmed Chrome via CDP...'); + return chromium.connectOverCDP('http://127.0.0.1:9222'); + }, + launchPersistentContext: async (_ud: string, _options: any) => { + consoleLogger.info('[SafeBrowsing] Connecting to pre-warmed Chrome via CDP...'); + const b = await chromium.connectOverCDP('http://127.0.0.1:9222'); + const ctx = b.contexts()[0]; + if (!ctx) throw new Error('[SafeBrowsing] No default context in pre-warmed Chrome'); + + const myPages = new Set(); + const origNewPage: any = ctx.newPage.bind(ctx); + ctx.newPage = async (options?: any) => { + const page = await origNewPage(options); + myPages.add(page); + page.once('close', () => myPages.delete(page)); + return page; + }; + + const origClose = ctx.close; + ctx.close = async () => { + for (const p of myPages) { + if (!p.isClosed()) await p.close().catch(() => {}); + } + myPages.clear(); + ctx.close = origClose; + ctx.newPage = origNewPage; + await b.close().catch(() => {}); + }; + + return ctx; + }, + name: () => 'chromium', + executablePath: () => '', + }; + consoleLogger.info('[SafeBrowsing] CDP launcher configured for pre-warmed Chrome on port 9222'); + return cdpLauncher; +}; + /** * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser) * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch */ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { const channel = browser || undefined; + const isLinuxDockerChrome = + browser === BrowserTypes.CHROME && process.platform === 'linux' && fs.existsSync('/.dockerenv'); const resolution = proxyInfoToResolution(cacheProxyInfo); const shouldIgnoreMuteAudio = @@ -2161,6 +2362,22 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { // during long crawls with multiple pool rotations finalArgs.push('--disk-cache-size=10485760'); + // Chrome under Linux Docker emulation can crash in GPU process startup. + // Apply a conservative software-only launch profile while keeping Chrome channel. + if (isLinuxDockerChrome) { + const dockerChromeArgs = [ + '--in-process-gpu', + '--disable-gpu-compositing', + '--disable-software-rasterizer', + '--disable-features=VizDisplayCompositor', + '--no-zygote', + ]; + + dockerChromeArgs.forEach(arg => { + if (!finalArgs.includes(arg)) finalArgs.push(arg); + }); + } + // Prevent Windows from throttling background Chromium processes if (os.platform() === 'win32') { finalArgs.push('--disable-features=UseEcoQoSForBackgroundProcess'); @@ -2187,12 +2404,39 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { break; } + const safeBrowsingEnabled = !!process.env.GOOGLE_SAFE_BROWSING; + + const baseIgnoredArgs = shouldIgnoreMuteAudio + ? ['--use-mock-keychain', '--mute-audio'] + : ['--use-mock-keychain']; + + if (isLinuxDockerChrome && !baseIgnoredArgs.includes('--enable-unsafe-swiftshader')) { + baseIgnoredArgs.push('--enable-unsafe-swiftshader'); + } + + const safeBrowsingIgnoredArgs = safeBrowsingEnabled + ? [ + '--safebrowsing-disable-auto-update', + '--disable-client-side-phishing-detection', + '--disable-background-networking', + '--disable-component-update', + ] + : []; + + if (safeBrowsingEnabled) { + finalArgs.push('--enable-features=SafeBrowsingEnhancedProtection'); + } + + let headless = process.env.CRAWLEE_HEADLESS === '1'; + if (safeBrowsingEnabled && headless && process.env.DISPLAY) { + headless = false; + consoleLogger.info(`[SafeBrowsing] Forcing headful mode (DISPLAY=${process.env.DISPLAY})`); + } + const options: LaunchOptions = { - ignoreDefaultArgs: shouldIgnoreMuteAudio - ? ['--use-mock-keychain', '--mute-audio'] - : ['--use-mock-keychain'], + ignoreDefaultArgs: [...baseIgnoredArgs, ...safeBrowsingIgnoredArgs], args: finalArgs, - headless: process.env.CRAWLEE_HEADLESS === '1', + headless, ...(channel && { channel }), ...(proxyOpt ? { proxy: proxyOpt } : {}), }; diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 387d1be8..c193a9f1 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -63,6 +63,16 @@ export const destinationPath = (storagePath: string): string => `${storagePath}/ */ export const getDefaultChromeDataDir = (): string => { try { + // If GOOGLE_SAFE_BROWSING is set, use the pre-warmed profile prepared by Dockerfile + if (process.env.GOOGLE_SAFE_BROWSING && fs.existsSync('/data/chrome-profile')) { + return '/data/chrome-profile'; + } + + // Check for environment override (used when GSB profile is pre-warmed in Docker) + if (process.env.OOBEE_CHROME_DATA_DIR && fs.existsSync(process.env.OOBEE_CHROME_DATA_DIR)) { + return process.env.OOBEE_CHROME_DATA_DIR; + } + let defaultChromeDataDir = null; if (os.platform() === 'win32') { defaultChromeDataDir = path.join( @@ -86,6 +96,26 @@ export const getDefaultChromeDataDir = (): string => { if (defaultChromeDataDir && fs.existsSync(defaultChromeDataDir)) { return defaultChromeDataDir; } + + // Linux: check if Chrome is installed; use same scratch dir pattern as Chromium + if (os.platform() === 'linux') { + const chromeExists = fs.existsSync('/usr/bin/google-chrome') || fs.existsSync('/usr/bin/google-chrome-stable'); + if (chromeExists) { + let linuxChromeDataDir = path.join(process.cwd(), 'Chromium Support'); + try { + fs.mkdirSync(linuxChromeDataDir, { recursive: true }); + } catch { + linuxChromeDataDir = '/tmp'; + } + // Create minimal Local State file so cloneChromeProfiles succeeds + const localStatePath = path.join(linuxChromeDataDir, 'Local State'); + if (!fs.existsSync(localStatePath)) { + fs.writeFileSync(localStatePath, JSON.stringify({ profile: { info_cache: {} } })); + } + return linuxChromeDataDir; + } + } + return null; } catch (error) { console.error(`Error in getDefaultChromeDataDir(): ${error}`); @@ -1000,6 +1030,7 @@ export const STATUS_CODE_METADATA: Record = { 0: 'Page Excluded', 1: 'Not A Supported Document', 2: 'Web Crawler Errored', + 3: 'Blocked by Safe Browsing', // 599 is set because Crawlee returns response status 100, 102, 103 as 599 599: 'Uncommon Response Status Code Received', diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index 7991e2e9..505b4927 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -23,6 +23,7 @@ import xPathToCss from './custom/xPathToCss.js'; import type { Response as PlaywrightResponse } from 'playwright'; import fs from 'fs'; import { getStoragePath } from '../utils.js'; +import { injectSafeBrowsingDb } from '../safeBrowsingProfile.js'; import path from 'path'; // types @@ -1265,6 +1266,14 @@ export const getPreLaunchHook = (userDataDirectory: string) => { const destProfile = path.join(effectiveDir, profile.name); await fsp.mkdir(destProfile, { recursive: true }).catch(() => {}); + // Copy Preferences so the Safe Browsing OHTTP key (hash_real_time_ohttp_key) + // is inherited from the base profile. injectSafeBrowsingDb() will merge + // enabled/enhanced on top, preserving the warmed-up key. + const prefsSrc = path.join(srcProfile, 'Preferences'); + if (await fsp.stat(prefsSrc).catch(() => null)) { + await fsp.copyFile(prefsSrc, path.join(destProfile, 'Preferences')).catch(() => {}); + } + // Cookies (macOS layout: /Cookies) const cookiesSrc = path.join(srcProfile, 'Cookies'); if (await fsp.stat(cookiesSrc).catch(() => null)) { @@ -1300,6 +1309,12 @@ export const getPreLaunchHook = (userDataDirectory: string) => { // Silent fallback: use empty profile if clone fails } + // Inject Safe Browsing preferences after copying base profile files. + // This merges enabled/enhanced on top of any copied Preferences, preserving + // the hash_real_time_ohttp_key from the base profile so Chrome does not need + // to re-fetch it for every pool browser. + injectSafeBrowsingDb(effectiveDir); + // Clean any stale lock files that may block browser launches on Windows const lockFiles = [ path.join(effectiveDir, 'SingletonLock'), @@ -1343,6 +1358,7 @@ export const getPostPageCloseHook = (userDataDirectory: string) => { }; }; + export const failedRequestHandler = async ({ request }: { request: Request }) => { guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url }); log.error(`Failed Request - ${request.url}: ${request.errorMessages}`); diff --git a/src/crawlers/crawlDomain.ts b/src/crawlers/crawlDomain.ts index 2c1f3b19..865e8376 100644 --- a/src/crawlers/crawlDomain.ts +++ b/src/crawlers/crawlDomain.ts @@ -26,6 +26,7 @@ import constants, { } from '../constants/constants.js'; import { getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, @@ -394,19 +395,21 @@ const crawlDomain = async ({ const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders); + const cdpLauncher = await getSafeBrowsingCdpLauncher(browser, userDataDirectory); + const crawler = register( new crawlee.PlaywrightCrawler({ launchContext: { - launcher: constants.launcher, + launcher: (cdpLauncher || constants.launcher) as any, launchOptions: getPlaywrightLaunchOptions(browser), }, retryOnBlocked: false, browserPoolOptions: { useFingerprints: false, - retireBrowserAfterPageCount: 500, + retireBrowserAfterPageCount: cdpLauncher ? Number.MAX_SAFE_INTEGER : 500, closeInactiveBrowserAfterSecs: 30, preLaunchHooks: [ - getPreLaunchHook(userDataDirectory), + ...(!cdpLauncher ? [getPreLaunchHook(userDataDirectory)] : []), async (_pageId, launchContext) => { // eslint-disable-next-line no-param-reassign launchContext.launchOptions = { @@ -420,7 +423,7 @@ const crawlDomain = async ({ }; }, ], - postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)], + postPageCloseHooks: [...(!cdpLauncher ? [getPostPageCloseHook(userDataDirectory)] : [])], }, requestQueue, maxRequestRetries: 3, @@ -522,6 +525,22 @@ const crawlDomain = async ({ actualUrl = page.url(); } + if (actualUrl.startsWith('chrome-error:')) { + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING; + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: isSafeBrowsingBlock ? STATUS_CODE_METADATA[3] : STATUS_CODE_METADATA[1], + httpStatusCode: isSafeBrowsingBlock ? 3 : 1, + }); + return; + } + // Second-pass requests: only do click-discovery, skip scanning if (request.label?.startsWith('__clickpass__')) { await enqueueProcess(page, enqueueLinks, browserContext); @@ -881,6 +900,27 @@ const crawlDomain = async ({ return; } + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING && + request.errorMessages?.some((msg: string) => + msg.includes('ERR_BLOCKED_BY_CLIENT') || + msg.includes('ERR_BLOCKED_BY_RESPONSE'), + ); + + if (isSafeBrowsingBlock) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: urlsCrawled.scanned.length, urlScanned: request.url, diff --git a/src/crawlers/crawlIntelligentSitemap.ts b/src/crawlers/crawlIntelligentSitemap.ts index 2fcccb77..ee1a25af 100644 --- a/src/crawlers/crawlIntelligentSitemap.ts +++ b/src/crawlers/crawlIntelligentSitemap.ts @@ -7,7 +7,7 @@ import { consoleLogger, guiInfoLog } from '../logs.js'; import crawlDomain from './crawlDomain.js'; import crawlSitemap from './crawlSitemap.js'; import { ViewportSettingsClass } from '../combine.js'; -import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent } from '../constants/common.js'; +import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent, launchPersistentSafeContext } from '../constants/common.js'; import { register } from '../utils.js'; const crawlIntelligentSitemap = async ( @@ -65,7 +65,7 @@ const crawlIntelligentSitemap = async ( if (process.env.CRAWLEE_HEADLESS === '1') { const effectiveUserDataDirectory = userDataDirectory || ''; - context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, { + context = await launchPersistentSafeContext(effectiveUserDataDirectory, { ...launchOptions, ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), ...(httpCredentials && { httpCredentials }), diff --git a/src/crawlers/crawlLocalFile.ts b/src/crawlers/crawlLocalFile.ts index 5e6d1742..83b11d6a 100644 --- a/src/crawlers/crawlLocalFile.ts +++ b/src/crawlers/crawlLocalFile.ts @@ -16,6 +16,7 @@ import { isFilePath, convertLocalFileToPath, convertPathToLocalFile, + launchPersistentSafeContext, } from '../constants/common.js'; import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js'; import { guiInfoLog } from '../logs.js'; @@ -151,7 +152,7 @@ export const crawlLocalFile = async ({ const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : ''; - const browserContext = await constants.launcher.launchPersistentContext( + const browserContext = await launchPersistentSafeContext( effectiveUserDataDirectory, { headless: process.env.CRAWLEE_HEADLESS === '1', diff --git a/src/crawlers/crawlSitemap.ts b/src/crawlers/crawlSitemap.ts index 8e2fc368..6374800b 100644 --- a/src/crawlers/crawlSitemap.ts +++ b/src/crawlers/crawlSitemap.ts @@ -24,6 +24,7 @@ import constants, { import { getLinksFromSitemap, getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, isDisallowedInRobotsTxt, isSkippedUrl, waitForPageLoaded, @@ -144,19 +145,21 @@ const crawlSitemap = async ({ // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery. const { requestQueue } = await createCrawleeSubFolders(randomToken); + const cdpLauncher = await getSafeBrowsingCdpLauncher(browser, userDataDirectory); + const crawler = register( new crawlee.PlaywrightCrawler({ launchContext: { - launcher: constants.launcher, + launcher: (cdpLauncher || constants.launcher) as any, launchOptions: getPlaywrightLaunchOptions(browser), }, retryOnBlocked: false, browserPoolOptions: { useFingerprints: false, - retireBrowserAfterPageCount: 500, + retireBrowserAfterPageCount: cdpLauncher ? Number.MAX_SAFE_INTEGER : 500, closeInactiveBrowserAfterSecs: 30, preLaunchHooks: [ - getPreLaunchHook(userDataDirectory), + ...(!cdpLauncher ? [getPreLaunchHook(userDataDirectory)] : []), async (_pageId, launchContext) => { launchContext.launchOptions = { ...launchContext.launchOptions, @@ -169,7 +172,7 @@ const crawlSitemap = async ({ }; }, ], - postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)], + postPageCloseHooks: [...(!cdpLauncher ? [getPostPageCloseHook(userDataDirectory)] : [])], }, requestList, requestQueue, @@ -275,6 +278,21 @@ const crawlSitemap = async ({ const actualUrl = page.url() || request.loadedUrl || request.url; + if (actualUrl.startsWith('chrome-error:')) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000; @@ -574,6 +592,27 @@ const crawlSitemap = async ({ return; } + const isSafeBrowsingBlock = !!process.env.GOOGLE_SAFE_BROWSING && + request.errorMessages?.some((msg: string) => + msg.includes('ERR_BLOCKED_BY_CLIENT') || + msg.includes('ERR_BLOCKED_BY_RESPONSE'), + ); + + if (isSafeBrowsingBlock) { + guiInfoLog(guiInfoStatusTypes.SKIPPED, { + numScanned: urlsCrawled.scanned.length, + urlScanned: request.url, + }); + urlsCrawled.userExcluded.push({ + url: request.url, + pageTitle: request.url, + actualUrl: request.url, + metadata: STATUS_CODE_METADATA[3], + httpStatusCode: 3, + }); + return; + } + guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: urlsCrawled.scanned.length, urlScanned: request.url, diff --git a/src/crawlers/custom/utils.ts b/src/crawlers/custom/utils.ts index b98cc735..4a9d7bb1 100644 --- a/src/crawlers/custom/utils.ts +++ b/src/crawlers/custom/utils.ts @@ -310,7 +310,12 @@ export const addOverlayMenu = async ( collapsed: false, }, ) => { - await page.waitForLoadState('domcontentloaded', { timeout: OVERLAY_OPERATION_TIMEOUT_MS }); + try { + await page.waitForLoadState('domcontentloaded', { timeout: 2000 }); + } catch { + // In CDP mode the load state may not resolve after script injection (e.g. axe-core). + // Proceed with injection anyway — the DOM is accessible if evaluate() succeeds. + } consoleLogger.info(`Overlay menu: adding to ${menuPos}...`); // Add the overlay menu with initial styling @@ -1211,11 +1216,19 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa .then(async () => { if (refreshSeq !== overlayRefreshSeq || page.isClosed()) return; + // During an active scan, navigation events (framenavigated/domcontentloaded) can fire + // due to axe-core injection or page resource loading. In CDP mode, concurrent + // page.evaluate() calls conflict with the running scan. Skip overlay injection + // for non-scan triggers while scanning — the overlay will be re-added by the + // 'scan-click' trigger after the scan completes. + if (pagesDict[pageId]?.isScanning && trigger !== 'scan-click') return; + try { // `framenavigated` can fire before the new document is ready for DOM inspection/injection. - await page.waitForLoadState('domcontentloaded', { timeout: 5000 }); + // Use a short timeout — in CDP mode, waitForLoadState can hang after script injection. + await page.waitForLoadState('domcontentloaded', { timeout: 2000 }); } catch { - // Best effort only. The page may still be mid-navigation. + // Best effort only. The page may still be mid-navigation or state tracking confused. } try { @@ -1306,6 +1319,18 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa if (page.isClosed()) return; await reconcileOverlayMenu('scan-click'); + + // If the overlay still isn't present after the first attempt (can happen in + // CDP mode where waitForLoadState tracking is unreliable), retry once. + if (!page.isClosed()) { + const overlayPresent = await page.evaluate(() => + Boolean(document.querySelector('#oobeeShadowHost')), + ).catch(() => false); + if (!overlayPresent) { + log('Overlay missing after scan-click reconcile, retrying...'); + await reconcileOverlayMenu('scan-click'); + } + } } catch (error) { log(`Scan failed ${error}`); } diff --git a/src/crawlers/guards/urlGuard.ts b/src/crawlers/guards/urlGuard.ts index c4932a59..1c694489 100644 --- a/src/crawlers/guards/urlGuard.ts +++ b/src/crawlers/guards/urlGuard.ts @@ -1,7 +1,11 @@ const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']); export function addUrlGuardScript(context, opts = {}) { - const { fallbackUrl }: any = opts; + const { fallbackUrl, allowChromeErrors }: any = opts; + + const allowedProtocols = allowChromeErrors + ? new Set([...ALLOWED_PROTOCOLS, 'chrome-error:']) + : ALLOWED_PROTOCOLS; const lastAllowedUrlByPage = new WeakMap(); @@ -64,7 +68,7 @@ export function addUrlGuardScript(context, opts = {}) { return restoreToSafeUrl(page, urlStr); } - if (ALLOWED_PROTOCOLS.has(urlObj.protocol)) { + if (allowedProtocols.has(urlObj.protocol)) { lastAllowedUrlByPage.set(page, urlObj.toString()); return; } diff --git a/src/crawlers/runCustom.ts b/src/crawlers/runCustom.ts index a3b8ae3c..c4dd1e06 100644 --- a/src/crawlers/runCustom.ts +++ b/src/crawlers/runCustom.ts @@ -7,13 +7,15 @@ import constants, { UrlsCrawled, } from '../constants/constants.js'; import { DEBUG, initNewPage, log } from './custom/utils.js'; -import { guiInfoLog } from '../logs.js'; +import { consoleLogger, guiInfoLog } from '../logs.js'; import { ViewportSettingsClass } from '../combine.js'; import { addUrlGuardScript } from './guards/urlGuard.js'; import { getBrowserToRun, getPlaywrightLaunchOptions, + getSafeBrowsingCdpLauncher, initModifiedUserAgent, + launchPersistentSafeContext, } from '../constants/common.js'; import { BrowserTypes } from '../constants/constants.js'; @@ -112,18 +114,33 @@ const runCustom = async ( const { authHeader, nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders); - const context = await constants.launcher.launchPersistentContext(userDataDirectory, { - ...baseLaunchOptions, - args: mergedArgs, - headless: false, - ignoreHTTPSErrors: true, - serviceWorkers: 'block', - viewport: null, - ...(hasCustomViewport ? contextDeviceOptions : {}), - userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined), - ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), - ...(httpCredentials && { httpCredentials }), - }); + const cdpLauncher = await getSafeBrowsingCdpLauncher(resolvedBrowserToRun, userDataDirectory); + let context; + let usedCdp = false; + + if (cdpLauncher) { + try { + context = await cdpLauncher.launchPersistentContext(userDataDirectory, {}); + usedCdp = true; + } catch (e) { + consoleLogger.info(`[SafeBrowsing] CDP fallback: ${e}`); + } + } + + if (!context) { + context = await launchPersistentSafeContext(userDataDirectory, { + ...baseLaunchOptions, + args: mergedArgs, + headless: false, + ignoreHTTPSErrors: true, + serviceWorkers: 'block' as const, + viewport: null, + ...(hasCustomViewport ? contextDeviceOptions : {}), + userAgent: process.env.OOBEE_USER_AGENT || (deviceUserAgent as string | undefined), + ...(nonAuthHeaders && { extraHTTPHeaders: nonAuthHeaders }), + ...(httpCredentials && { httpCredentials }), + }); + } if (authHeader) { await addAuthRouteHandler(context, url, authHeader); @@ -140,9 +157,11 @@ const runCustom = async ( // For handling closing playwright browser and continue generate artifacts etc registerSoftClose(processPageParams.stopAll); - addUrlGuardScript(context, { fallbackUrl: url }); + addUrlGuardScript(context, { fallbackUrl: url, allowChromeErrors: !!process.env.GOOGLE_SAFE_BROWSING }); - const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage()); + const page = usedCdp + ? await context.newPage() + : context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage()); await initNewPage(page, pageClosePromises, processPageParams, pagesDict); // Detection of new page diff --git a/src/npmIndex.ts b/src/npmIndex.ts index 7d1b252d..f72e061d 100644 --- a/src/npmIndex.ts +++ b/src/npmIndex.ts @@ -10,6 +10,7 @@ import { deleteClonedProfiles, getBrowserToRun, getPlaywrightLaunchOptions, + launchPersistentSafeContext, submitForm, } from './constants/common.js'; import { createCrawleeSubFolders, enrichViolationMessages, filterAxeResults } from './crawlers/commonCrawlerFunc.js'; @@ -433,7 +434,7 @@ export const init = async ({ browserToRun = browserData.browserToRun; clonedBrowserDataDir = browserData.clonedBrowserDataDir; - browserContext = await constants.launcher.launchPersistentContext(clonedBrowserDataDir, { + browserContext = await launchPersistentSafeContext(clonedBrowserDataDir, { viewport: viewportSettings, ...getPlaywrightLaunchOptions(browserToRun), }); diff --git a/src/safeBrowsingProfile.ts b/src/safeBrowsingProfile.ts new file mode 100644 index 00000000..41faf300 --- /dev/null +++ b/src/safeBrowsingProfile.ts @@ -0,0 +1,278 @@ +import { type ChildProcess, execSync, spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import printMessage from 'print-message'; +import { chromium as playwrightChromium } from 'playwright'; +import { consoleLogger } from './logs.js'; +import { messageOptions } from './constants/common.js'; + +const BASE_PROFILE_DIR = path.join(os.homedir(), '.oobee', 'safe-browsing-profile'); +const SB_DIR = path.join(BASE_PROFILE_DIR, 'Safe Browsing'); +const SEEDED_MARKER = '.sb-seeded'; +const LOCK_DIR = path.join(BASE_PROFILE_DIR, '.warmup-lock'); +const LOCK_STALE_MS = 180_000; + +function getChromeExecutable(): string { + const candidates: string[] = + process.platform === 'darwin' + ? ['/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'] + : [ + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + ]; + + const found = candidates.find(p => fs.existsSync(p)); + if (found) return found; + + try { + const playwrightPath = playwrightChromium.executablePath(); + if (fs.existsSync(playwrightPath)) return playwrightPath; + } catch {} + + return 'google-chrome'; +} + +function findSystemSafeBrowsingDir(): string | null { + const candidates = + process.platform === 'darwin' + ? [path.join(os.homedir(), 'Library', 'Application Support', 'Google', 'Chrome', 'Safe Browsing')] + : [ + '/opt/oobee-safe-browsing/Safe Browsing', + path.join(os.homedir(), '.config', 'google-chrome', 'Safe Browsing'), + path.join(os.homedir(), '.config', 'chromium', 'Safe Browsing'), + ]; + return candidates.find(isDbDir) ?? null; +} + +function isDbDir(dir: string): boolean { + if (!fs.existsSync(dir)) return false; + return fs.readdirSync(dir).some(f => f.startsWith('UrlSoceng.store.') || f.startsWith('UrlMalware.store.')); +} + +function copyDirectory(src: string, dst: string): void { + fs.mkdirSync(dst, { recursive: true }); + for (const file of fs.readdirSync(src)) { + fs.copyFileSync(path.join(src, file), path.join(dst, file)); + } +} + +function killChromeTree(chrome: ChildProcess): void { + if (!chrome.pid) return; + try { + process.kill(-chrome.pid, 'SIGKILL'); + } catch { + try { chrome.kill('SIGKILL'); } catch {} + } +} + +function acquireLock(): boolean { + try { + fs.mkdirSync(LOCK_DIR); + fs.writeFileSync(path.join(LOCK_DIR, 'pid'), `${process.pid}\n${Date.now()}`); + return true; + } catch { + try { + const content = fs.readFileSync(path.join(LOCK_DIR, 'pid'), 'utf8'); + const timestamp = parseInt(content.split('\n')[1], 10); + if (Date.now() - timestamp > LOCK_STALE_MS) { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + return acquireLock(); + } + } catch { + fs.rmSync(LOCK_DIR, { recursive: true, force: true }); + return acquireLock(); + } + return false; + } +} + +function releaseLock(): void { + try { fs.rmSync(LOCK_DIR, { recursive: true, force: true }); } catch {} +} + +async function spawnChromeForWarmup(): Promise { + printMessage(['Downloading Safe Browsing threat database via Chrome (up to 120s)...'], messageOptions); + + fs.mkdirSync(path.join(BASE_PROFILE_DIR, 'Default'), { recursive: true }); + fs.writeFileSync( + path.join(BASE_PROFILE_DIR, 'Default', 'Preferences'), + JSON.stringify({ safebrowsing: { enabled: true, enhanced: true } }), + ); + + const exe = getChromeExecutable(); + + const baseArgs = [ + `--user-data-dir=${BASE_PROFILE_DIR}`, + '--no-first-run', + '--no-default-browser-check', + '--disable-extensions', + ...(process.platform === 'linux' ? ['--no-sandbox', '--disable-setuid-sandbox'] : []), + ]; + + let spawnEnv: NodeJS.ProcessEnv = { ...process.env }; + let windowArgs: string[] = ['--window-position=-10000,-10000', '--window-size=1,1']; + let xvfbProcess: ChildProcess | null = null; + + if (process.platform === 'linux' && !process.env.DISPLAY) { + const displayNum = ':99'; + let xvfbStarted = false; + try { + xvfbProcess = spawn('Xvfb', [displayNum, '-screen', '0', '1024x768x24'], { + stdio: 'ignore', + detached: true, + }); + xvfbProcess.unref(); + await new Promise(r => setTimeout(r, 1500)); + + if (xvfbProcess.exitCode === null && !xvfbProcess.killed) { + spawnEnv = { ...process.env, DISPLAY: displayNum }; + xvfbStarted = true; + } else { + xvfbProcess = null; + } + } catch { + xvfbProcess = null; + } + + if (!xvfbStarted) { + windowArgs = ['--headless=old', '--disable-gpu']; + } + } + + const chrome = spawn( + exe, + [...baseArgs, ...windowArgs, 'about:blank'], + { stdio: 'ignore', detached: true, env: spawnEnv }, + ); + + const maxWait = 120_000; + const pollInterval = 5_000; + let waited = 0; + while (!isDbDir(SB_DIR) && waited < maxWait) { + await new Promise(r => setTimeout(r, pollInterval)); + waited += pollInterval; + } + + killChromeTree(chrome); + + if (xvfbProcess?.pid) { + try { process.kill(xvfbProcess.pid, 'SIGTERM'); } catch {} + } +} + +export async function warmupSafeBrowsingBaseProfile(): Promise { + consoleLogger.info(`[SafeBrowsing] BASE_PROFILE_DIR: ${BASE_PROFILE_DIR}`); + consoleLogger.info(`[SafeBrowsing] SB_DIR: ${SB_DIR}`); + consoleLogger.info(`[SafeBrowsing] isDbDir(SB_DIR): ${isDbDir(SB_DIR)}`); + if (isDbDir(SB_DIR)) { + consoleLogger.info('[SafeBrowsing] DB already exists in base profile, skipping warmup'); + return; + } + + fs.mkdirSync(BASE_PROFILE_DIR, { recursive: true }); + + const systemSbDir = findSystemSafeBrowsingDir(); + consoleLogger.info(`[SafeBrowsing] findSystemSafeBrowsingDir() = ${systemSbDir}`); + if (systemSbDir) { + consoleLogger.info(`[SafeBrowsing] Found system DB at: ${systemSbDir}`); + const files = fs.readdirSync(systemSbDir); + consoleLogger.info(`[SafeBrowsing] Files in system DB: ${files.join(', ')}`); + printMessage(['Copying Safe Browsing threat database from system Chrome profile...'], messageOptions); + copyDirectory(systemSbDir, SB_DIR); + printMessage(['Google Safe Browsing enabled (real-time URL protection active)'], messageOptions); + return; + } + + const exe = getChromeExecutable(); + const chromeFound = fs.existsSync(exe); + consoleLogger.info(`[SafeBrowsing] Chrome executable: ${exe}, found: ${chromeFound}`); + if (!chromeFound) { + printMessage(['WARNING: Google Chrome not found. Safe Browsing requires Chrome (not Chromium). On Linux Docker, build with --platform linux/amd64.'], messageOptions); + return; + } + + // Chrome 128+ uses real-time v5 hash-prefix lookups via OHTTP and no longer + // downloads local UrlSoceng.store.* files. Skip the legacy DB download and + // rely on real-time API with preferences set by injectSafeBrowsingDb(). + consoleLogger.info('[SafeBrowsing] Skipping legacy DB download (Chrome 128+ uses real-time v5 API)'); + printMessage(['Google Safe Browsing enabled (real-time URL protection via Chrome)'], messageOptions); + return; + + if (!acquireLock()) { + consoleLogger.info('Another process is downloading Safe Browsing DB; waiting...'); + const waitStart = Date.now(); + while (!isDbDir(SB_DIR) && Date.now() - waitStart < 150_000) { + await new Promise(r => setTimeout(r, 5_000)); + } + return; + } + + try { + await spawnChromeForWarmup(); + + if (isDbDir(SB_DIR)) { + printMessage(['Google Safe Browsing enabled (real-time URL protection active)'], messageOptions); + } else { + printMessage(['WARNING: Safe Browsing DB did not populate. Protection may be reduced.'], messageOptions); + } + } finally { + releaseLock(); + } +} + +export function injectSafeBrowsingDb(targetDir: string): void { + consoleLogger.info(`[SafeBrowsing] injectSafeBrowsingDb(${targetDir})`); + consoleLogger.info(`[SafeBrowsing] isDbDir(SB_DIR=${SB_DIR}): ${isDbDir(SB_DIR)}`); + if (!isDbDir(SB_DIR)) { + consoleLogger.info('[SafeBrowsing] No DB to inject — setting preferences only'); + const defaultDir = path.join(targetDir, 'Default'); + fs.mkdirSync(defaultDir, { recursive: true }); + const prefsPath = path.join(defaultDir, 'Preferences'); + let prefs: Record = {}; + if (fs.existsSync(prefsPath)) { + try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} + } + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: true }; + fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + consoleLogger.info(`[SafeBrowsing] Wrote preferences to ${prefsPath}: ${JSON.stringify(prefs.safebrowsing)}`); + return; + } + if (fs.existsSync(path.join(targetDir, SEEDED_MARKER))) { + consoleLogger.info('[SafeBrowsing] Already seeded (marker exists), skipping'); + return; + } + + consoleLogger.info('[SafeBrowsing] Copying DB + setting preferences'); + copyDirectory(SB_DIR, path.join(targetDir, 'Safe Browsing')); + + const defaultDir = path.join(targetDir, 'Default'); + fs.mkdirSync(defaultDir, { recursive: true }); + const prefsPath = path.join(defaultDir, 'Preferences'); + let prefs: Record = {}; + if (fs.existsSync(prefsPath)) { + try { prefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8')); } catch {} + } + prefs.safebrowsing = { ...(prefs.safebrowsing as object), enabled: true, enhanced: true }; + fs.writeFileSync(prefsPath, JSON.stringify(prefs)); + consoleLogger.info(`[SafeBrowsing] Wrote preferences to ${prefsPath}: ${JSON.stringify(prefs.safebrowsing)}`); + + fs.writeFileSync(path.join(targetDir, SEEDED_MARKER), new Date().toISOString()); + consoleLogger.info('[SafeBrowsing] Injection complete'); +} + +export async function ensureAndInjectSafeBrowsing(targetDir: string): Promise { + if (!process.env.GOOGLE_SAFE_BROWSING) return; + consoleLogger.info(`[SafeBrowsing] ensureAndInjectSafeBrowsing(${targetDir})`); + + if (process.platform === 'win32') { + printMessage(['Google Safe Browsing is not yet supported on Windows.'], messageOptions); + return; + } + + await warmupSafeBrowsingBaseProfile(); + injectSafeBrowsingDb(targetDir); + consoleLogger.info('[SafeBrowsing] ensureAndInjectSafeBrowsing complete'); +}