chore: stop standby requests from spawning duplicate crawlers

src/const.ts

@@ -18,6 +18,8 @@ export enum ContentCrawlerTypes {
     CHEERIO = 'cheerio',
 }
 
+export type CrawlerKind = 'search' | ContentCrawlerTypes;
+
 export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60;
 
 export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10;

src/crawlers.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 
 import { ImpitHttpClient } from '@crawlee/impit-client';
@@ -16,12 +17,19 @@ import {
     type RequestOptions,
 } from 'crawlee';
 
+import type { CrawlerKind } from './const.js';
 import { ContentCrawlerTypes, GOOGLE_STANDARD_RESULTS_PER_PAGE } from './const.js';
 import { deduplicateResults, scrapeOrganicResults } from './google-search/google-extractors-urls.js';
 import { getMiniActor } from './mini-actors.js';
 import { failedRequestHandler, requestHandlerCheerio, requestHandlerPlaywright } from './request-handler.js';
 import { addEmptyResultToResponse, sendResponseError } from './responses.js';
-import type { ContentCrawlerOptions, ContentCrawlerUserData, SearchCrawlerUserData } from './types.js';
+import type {
+    ContentCrawlerOptions,
+    ContentCrawlerUserData,
+    ProxyOptions,
+    SearchCrawlerOptions,
+    SearchCrawlerUserData,
+} from './types.js';
 import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby, randomId } from './utils.js';
 
 const crawlers = new Map<string, CheerioCrawler | PlaywrightCrawler>();
@@ -50,8 +58,64 @@ async function getGhosteryBlocker(): Promise<PlaywrightBlocker | undefined> {
     }
 }
 
-export function getCrawlerKey(crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions) {
-    return JSON.stringify(crawlerOptions);
+/** Mirrors how `Actor.createProxyConfiguration` reads these options, so equivalent spellings share a crawler. */
+function resolveProxyOptions(proxyOptions: ProxyOptions) {
+    const {
+        useApifyProxy,
+        checkAccess,
+        newUrlFunction,
+        apifyProxyGroups,
+        apifyProxyCountry,
+        apifyProxySubdivision,
+        ...rest
+    } = proxyOptions;
+
+    if (useApifyProxy === false && !rest.proxyUrls) {
+        return null;
+    }
+
+    return {
+        ...rest,
+        groups: rest.groups?.length ? rest.groups : apifyProxyGroups,
+        countryCode: rest.countryCode || apifyProxyCountry,
+        subdivisionCode: rest.subdivisionCode || apifyProxySubdivision,
+    };
+}
+
+/** `JSON.stringify` with object keys sorted at every level. Array order is kept, it carries meaning. */
+function canonicalJson(value: unknown): string {
+    if (value === null || typeof value !== 'object') {
+        return JSON.stringify(value) ?? 'null';
+    }
+    if (Array.isArray(value)) {
+        return `[${value.map(canonicalJson).join(',')}]`;
+    }
+    const entries = Object.entries(value)
+        .filter(([, entryValue]) => entryValue !== undefined)
+        .sort(([a], [b]) => (a < b ? -1 : 1));
+    return `{${entries.map(([key, entryValue]) => `${JSON.stringify(key)}:${canonicalJson(entryValue)}`).join(',')}}`;
+}
+
+/**
+ * Identifies a crawler in the `crawlers` cache. Listed are the only options the builders in
+ * `input.ts` derive from the input; the proxy options stand in for the constructed
+ * `ProxyConfiguration`, whose child logger snapshots the log level and so kept changing the key.
+ * Hashed because the key is logged and used as a queue name, while the options can hold credentials.
+ */
+export function getCrawlerKey(
+    kind: CrawlerKind,
+    crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions,
+    proxyOptions: ProxyOptions,
+): string {
+    const fingerprint = {
+        keepAlive: crawlerOptions.keepAlive,
+        maxRequestRetries: crawlerOptions.maxRequestRetries,
+        requestHandlerTimeoutSecs: crawlerOptions.requestHandlerTimeoutSecs,
+        desiredConcurrency: crawlerOptions.autoscaledPoolOptions?.desiredConcurrency,
+        proxy: resolveProxyOptions(proxyOptions),
+    };
+    const hash = createHash('sha1').update(canonicalJson(fingerprint)).digest('hex');
+    return `${kind}-${hash.slice(0, 12)}`;
 }
 
 /**
@@ -86,17 +150,18 @@ export const addContentCrawlRequest = async (
  * A crawler won't be created if it already exists.
  */
 export async function createAndStartSearchCrawler(
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerOptions: SearchCrawlerOptions,
     startCrawler = true,
 ) {
-    const key = getCrawlerKey(searchCrawlerOptions);
+    const { crawlerOptions, proxyOptions } = searchCrawlerOptions;
+    const key = getCrawlerKey('search', crawlerOptions, proxyOptions);
     if (crawlers.has(key)) {
         return { key, crawler: crawlers.get(key) };
     }
 
     log.info(`Creating new cheerio crawler with key ${key}`);
     const crawler = new CheerioCrawler({
-        ...(searchCrawlerOptions as CheerioCrawlerOptions),
+        ...crawlerOptions,
         requestQueue: await RequestQueue.open(key, { storageClient: client }),
         requestHandler: async ({ request, $: _$, addRequests }: CheerioCrawlingContext<SearchCrawlerUserData>) => {
             // NOTE: we need to cast this to fix `cheerio` type errors
@@ -135,7 +200,7 @@ export async function createAndStartSearchCrawler(
                         collectedResults: deduplicated,
                         currentPage: nextPage,
                     },
-                    searchCrawlerOptions.proxyConfiguration,
+                    proxyOptions,
                     nextOffset,
                 );
                 await addRequests([nextRequest]);
@@ -190,9 +255,9 @@ export async function createAndStartContentCrawler(
     contentCrawlerOptions: ContentCrawlerOptions,
     startCrawler = true,
 ) {
-    const { type: crawlerType, crawlerOptions } = contentCrawlerOptions;
+    const { type: crawlerType, crawlerOptions, proxyOptions } = contentCrawlerOptions;
 
-    const key = getCrawlerKey(crawlerOptions);
+    const key = getCrawlerKey(crawlerType, crawlerOptions, proxyOptions);
     if (crawlers.has(key)) {
         return { key, crawler: crawlers.get(key) };
     }
@@ -325,18 +390,17 @@ async function maybeCharge(crawlerType: ContentCrawlerTypes, userAuthorization?:
 }
 
 /**
- * Adds a search request to the Google search crawler.
+ * Adds a search request to the Google search crawler identified by `searchCrawlerKey`.
  * Create a response for the request and set the desired number of results (maxResults).
  */
 export const addSearchRequest = async (
     request: RequestOptions<ContentCrawlerUserData>,
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerKey: string,
 ) => {
-    const key = getCrawlerKey(searchCrawlerOptions);
-    const crawler = crawlers.get(key);
+    const crawler = crawlers.get(searchCrawlerKey);
 
     if (!crawler) {
-        log.error(`Cheerio crawler not found: key ${key}`);
+        log.error(`Search crawler not found: key ${searchCrawlerKey}`);
         return;
     }
     addTimeMeasureEvent(request.userData!, 'before-cheerio-queue-add');

src/input.ts

@@ -1,6 +1,6 @@
 import type { ProxyConfigurationOptions } from 'apify';
 import { Actor } from 'apify';
-import type { CheerioCrawlerOptions, ProxyConfiguration } from 'crawlee';
+import type { ProxyConfiguration } from 'crawlee';
 import { BrowserName, log } from 'crawlee';
 import { firefox } from 'playwright';
 
@@ -14,8 +14,10 @@ import type {
     ContentScraperSettings,
     Input,
     OutputFormats,
+    ProxyOptions,
     RagWebBrowserInput,
     ScrapingTool,
+    SearchCrawlerOptions,
     SERPProxyGroup,
     UrlToMarkdownInput,
 } from './types.js';
@@ -59,7 +61,7 @@ async function processInputInternal(
 ) {
     const miniActor = getMiniActor();
     let input: Input;
-    let searchCrawlerOptions: CheerioCrawlerOptions = {};
+    let searchCrawlerOptions: SearchCrawlerOptions = { crawlerOptions: {}, proxyOptions: {} };
 
     if (miniActor.runsSearch) {
         const processedRagWebBrowserInput = await processRagWebBrowserInput(
@@ -79,8 +81,6 @@ async function processInputInternal(
         removeCookieWarnings,
     } = input;
 
-    log.setLevel(debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO);
-
     const contentScraperSettings: ContentScraperSettings = {
         debugMode,
         dynamicContentWaitSecs,
@@ -97,7 +97,7 @@ async function processInputInternal(
 async function processRagWebBrowserInput(input: Partial<RagWebBrowserInput>, standbyInit: boolean):
     Promise<{
         validatedRagBrowserInput: RagWebBrowserInput;
-        searchCrawlerOptions: CheerioCrawlerOptions
+        searchCrawlerOptions: SearchCrawlerOptions
     }> {
     /* eslint-disable no-param-reassign */
 
@@ -166,12 +166,16 @@ async function processRagWebBrowserInput(input: Partial<RagWebBrowserInput>, sta
         input.dynamicContentWaitSecs = Math.round(input.requestTimeoutSecs / 2);
     }
 
-    const proxySearch = await Actor.createProxyConfiguration({ groups: [input.serpProxyGroup], checkAccess: false });
-    const searchCrawlerOptions: CheerioCrawlerOptions = {
-        keepAlive: standbyInit,
-        maxRequestRetries: input.serpMaxRetries,
-        proxyConfiguration: proxySearch,
-        autoscaledPoolOptions: { desiredConcurrency: 1 },
+    const proxyOptions: ProxyOptions = { groups: [input.serpProxyGroup] };
+    const proxySearch = await Actor.createProxyConfiguration({ ...proxyOptions, checkAccess: false });
+    const searchCrawlerOptions: SearchCrawlerOptions = {
+        crawlerOptions: {
+            keepAlive: standbyInit,
+            maxRequestRetries: input.serpMaxRetries,
+            proxyConfiguration: proxySearch,
+            autoscaledPoolOptions: { desiredConcurrency: 1 },
+        },
+        proxyOptions,
     };
     const validatedRagBrowserInput = validateAndFillInput(input) as RagWebBrowserInput;
     return {
@@ -219,6 +223,7 @@ function createPlaywrightCrawlerOptions(
 
     return {
         type: ContentCrawlerTypes.PLAYWRIGHT,
+        proxyOptions: input.proxyConfiguration,
         crawlerOptions: {
             headless: true,
             keepAlive,
@@ -261,6 +266,7 @@ function createCheerioCrawlerOptions(
 
     return {
         type: ContentCrawlerTypes.CHEERIO,
+        proxyOptions: input.proxyConfiguration,
         crawlerOptions: {
             keepAlive,
             maxRequestRetries,

src/main.ts

@@ -18,6 +18,10 @@ Actor.on('migrating', () => {
 
 const originalInput = await Actor.getInput<Partial<Input>>() ?? {} as Input;
 
+// Set once: the level is process-wide, so a standby request changing it would leak debug output
+// across concurrent callers. Per-request `debugMode` still fills the response's `debug` field.
+log.setLevel(originalInput.debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO);
+
 if (isActorStandby()) {
     log.info('Actor is running in the STANDBY mode.');
 

src/search.ts

@@ -1,14 +1,22 @@
 import type { IncomingMessage, ServerResponse } from 'node:http';
 
-import { type CheerioCrawlerOptions, log } from 'crawlee';
+import { log } from 'crawlee';
 
 import { PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS } from './const.js';
 import { addContentCrawlRequest, addSearchRequest, createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js';
 import { UserInputError } from './errors.js';
 import { processInput } from './input.js';
 import { getMiniActor } from './mini-actors.js';
 import { createResponsePromise } from './responses.js';
-import type { ContentCrawlerOptions, ContentScraperSettings, Input, Output, RagWebBrowserInput, UrlToMarkdownInput } from './types.js';
+import type {
+    ContentCrawlerOptions,
+    ContentScraperSettings,
+    Input,
+    Output,
+    RagWebBrowserInput,
+    SearchCrawlerOptions,
+    UrlToMarkdownInput,
+} from './types.js';
 import {
     addTimeMeasureEvent,
     createRequest,
@@ -26,7 +34,7 @@ import {
  */
 function prepareRequest(
     input: Input,
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerOptions: SearchCrawlerOptions,
     contentCrawlerKey: string,
     contentScraperSettings: ContentScraperSettings,
     userAuthorization?: string,
@@ -80,7 +88,7 @@ function prepareRequest(
                 contentScraperSettings,
                 userAuthorization,
             },
-            searchCrawlerOptions.proxyConfiguration,
+            searchCrawlerOptions.proxyOptions,
         );
 
     addTimeMeasureEvent(req.userData!, 'request-received', Date.now());
@@ -101,7 +109,7 @@ async function runSearchProcess(params: Partial<Input>, userAuthorization?: stri
     } = await processInput(params);
 
     // Set keepAlive to true to find the correct crawlers
-    searchCrawlerOptions.keepAlive = true;
+    searchCrawlerOptions.crawlerOptions.keepAlive = true;
     contentCrawlerOptions.crawlerOptions.keepAlive = true;
 
     const { key: contentCrawlerKey } = await createAndStartContentCrawler(contentCrawlerOptions);
@@ -125,9 +133,9 @@ async function runSearchProcess(params: Partial<Input>, userAuthorization?: stri
         }
         await addContentCrawlRequest(req, responseId, contentCrawlerKey);
     } else {
-        await createAndStartSearchCrawler(searchCrawlerOptions);
         // If input is a search query, run the search crawler first
-        await addSearchRequest(req, searchCrawlerOptions);
+        const { key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions);
+        await addSearchRequest(req, searchCrawlerKey);
     }
 
     // Return promise that resolves when all requests are processed
@@ -178,7 +186,7 @@ export async function handleModelContextProtocol(params: Partial<Input>, userAut
  */
 export async function handleSearchNormalMode(
     input: Input,
-    searchCrawlerOptions: CheerioCrawlerOptions,
+    searchCrawlerOptions: SearchCrawlerOptions,
     contentCrawlerOptions: ContentCrawlerOptions,
     contentScraperSettings: ContentScraperSettings,
 ) {
@@ -205,8 +213,8 @@ export async function handleSearchNormalMode(
         }
         await addContentCrawlRequest(req, '', contentCrawlerKey);
     } else {
-        const { crawler: searchCrawler } = await createAndStartSearchCrawler(searchCrawlerOptions, false);
-        await addSearchRequest(req, searchCrawlerOptions);
+        const { crawler: searchCrawler, key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions, false);
+        await addSearchRequest(req, searchCrawlerKey);
         addTimeMeasureEvent(req.userData!, 'before-cheerio-run', startedTime);
         log.info(`Running Google Search crawler with request: ${JSON.stringify(req)}`);
         await searchCrawler!.run();

src/types.ts

@@ -12,6 +12,9 @@ import type { ContentCrawlerTypes } from './const.js';
  */
 type Optional<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
 
+/** What `Actor.createProxyConfiguration` accepts: the SDK options plus the input-schema flag. */
+export type ProxyOptions = ProxyConfigurationOptions & { useApifyProxy?: boolean };
+
 export type OutputFormats = 'text' | 'markdown' | 'html';
 export type SERPProxyGroup = 'GOOGLE_SERP' | 'SHADER';
 export type ScrapingTool = 'browser-playwright' | 'raw-http';
@@ -159,10 +162,18 @@ export type Output = {
     };
 };
 
+/** The proxy options are kept so nothing has to read them back off the constructed `ProxyConfiguration`. */
+export type SearchCrawlerOptions = {
+    crawlerOptions: CheerioCrawlerOptions;
+    proxyOptions: ProxyOptions;
+};
+
 export type ContentCrawlerOptions = {
     type: ContentCrawlerTypes.CHEERIO,
-    crawlerOptions: CheerioCrawlerOptions
+    crawlerOptions: CheerioCrawlerOptions,
+    proxyOptions: ProxyOptions,
 } | {
     type: ContentCrawlerTypes.PLAYWRIGHT,
-    crawlerOptions: PlaywrightCrawlerOptions
+    crawlerOptions: PlaywrightCrawlerOptions,
+    proxyOptions: ProxyOptions,
 };