From 2a05fdb4fa42859fa4fd757841a0bf079d3a7e92 Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Thu, 6 Aug 2026 16:29:31 +0200 Subject: [PATCH 1/2] fix: stop standby requests from spawning duplicate crawlers --- src/const.ts | 2 + src/crawlers.ts | 99 +++++++++++++++++--- src/input.ts | 28 +++--- src/main.ts | 5 + src/search.ts | 28 ++++-- src/types.ts | 19 +++- src/utils.ts | 10 +- tests/cheerio-crawler.content.test.ts | 2 +- tests/crawler-key.test.ts | 128 ++++++++++++++++++++++++++ 9 files changed, 278 insertions(+), 43 deletions(-) create mode 100644 tests/crawler-key.test.ts diff --git a/src/const.ts b/src/const.ts index f8baaa0..7c98f83 100644 --- a/src/const.ts +++ b/src/const.ts @@ -18,6 +18,8 @@ export enum ContentCrawlerTypes { CHEERIO = 'cheerio', } +export type CrawlerKind = 'search' | ContentCrawlerTypes; + export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60; export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10; diff --git a/src/crawlers.ts b/src/crawlers.ts index 9515bd3..581e225 100644 --- a/src/crawlers.ts +++ b/src/crawlers.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFile } from 'node:fs/promises'; import { ImpitHttpClient } from '@crawlee/impit-client'; @@ -16,12 +17,19 @@ import { type RequestOptions, } from 'crawlee'; +import type { CrawlerKind } from './const.js'; import { ContentCrawlerTypes, GOOGLE_STANDARD_RESULTS_PER_PAGE } from './const.js'; import { deduplicateResults, scrapeOrganicResults } from './google-search/google-extractors-urls.js'; import { getMiniActor } from './mini-actors.js'; import { failedRequestHandler, requestHandlerCheerio, requestHandlerPlaywright } from './request-handler.js'; import { addEmptyResultToResponse, sendResponseError } from './responses.js'; -import type { ContentCrawlerOptions, ContentCrawlerUserData, SearchCrawlerUserData } from './types.js'; +import type { + ContentCrawlerOptions, + ContentCrawlerUserData, + ProxyOptions, + SearchCrawlerOptions, + SearchCrawlerUserData, +} from './types.js'; import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby, randomId } from './utils.js'; const crawlers = new Map(); @@ -50,8 +58,71 @@ async function getGhosteryBlocker(): Promise { } } -export function getCrawlerKey(crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions) { - return JSON.stringify(crawlerOptions); +/** `checkAccess` only drives initialization, and no serialization tells two functions apart. */ +const PROXY_OPTIONS_EXCLUDED_FROM_KEY = new Set(['checkAccess', 'newUrlFunction']); + +/** + * Resolves the aliases that `ProxyConfiguration` itself resolves, so options that differ only in + * which spelling they use share one crawler instead of getting one each. + */ +function resolveProxyOptions(proxyOptions: ProxyOptions) { + const { apifyProxyGroups, apifyProxyCountry, apifyProxySubdivision, ...rest } = proxyOptions; + const resolved = { + ...rest, + useApifyProxy: rest.useApifyProxy !== false, + groups: rest.groups?.length ? rest.groups : apifyProxyGroups, + countryCode: rest.countryCode || apifyProxyCountry, + subdivisionCode: rest.subdivisionCode || apifyProxySubdivision, + }; + + return Object.fromEntries( + Object.entries(resolved).filter(([key]) => !PROXY_OPTIONS_EXCLUDED_FROM_KEY.has(key)), + ); +} + +/** + * `JSON.stringify` with object keys sorted at every level, so that two objects differing only in key + * order serialize identically. Array order is kept because it carries meaning, such as the rotation + * order of `proxyUrls`. `Date` and `toJSON` are not honoured; the input here is parsed JSON. + */ +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value) ?? 'null'; + } + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + const entries = Object.entries(value) + .filter(([, entryValue]) => entryValue !== undefined) + .sort(([a], [b]) => (a < b ? -1 : 1)); + return `{${entries.map(([key, entryValue]) => `${JSON.stringify(key)}:${canonicalJson(entryValue)}`).join(',')}}`; +} + +/** + * Identifies a crawler in the `crawlers` cache. The listed fields are the only ones the option + * builders in `input.ts` derive from the input; everything else they set is a constant. + * + * The proxy *options* stand in for the constructed `ProxyConfiguration`, whose child logger + * snapshots the log level at construction time. Serializing that instance made identical requests + * miss the cache: https://github.com/apify/actor-rag-web-browser/issues/60. + * + * The fingerprint is hashed because the key is logged and used as the request queue name, while the + * options may hold the proxy password or a custom proxy URL with credentials in it. + */ +export function getCrawlerKey( + kind: CrawlerKind, + crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions, + proxyOptions: ProxyOptions, +): string { + const fingerprint = { + keepAlive: crawlerOptions.keepAlive, + maxRequestRetries: crawlerOptions.maxRequestRetries, + requestHandlerTimeoutSecs: crawlerOptions.requestHandlerTimeoutSecs, + desiredConcurrency: crawlerOptions.autoscaledPoolOptions?.desiredConcurrency, + proxy: resolveProxyOptions(proxyOptions), + }; + const hash = createHash('sha1').update(canonicalJson(fingerprint)).digest('hex'); + return `${kind}-${hash.slice(0, 12)}`; } /** @@ -86,17 +157,18 @@ export const addContentCrawlRequest = async ( * A crawler won't be created if it already exists. */ export async function createAndStartSearchCrawler( - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, startCrawler = true, ) { - const key = getCrawlerKey(searchCrawlerOptions); + const { crawlerOptions, proxyOptions } = searchCrawlerOptions; + const key = getCrawlerKey('search', crawlerOptions, proxyOptions); if (crawlers.has(key)) { return { key, crawler: crawlers.get(key) }; } log.info(`Creating new cheerio crawler with key ${key}`); const crawler = new CheerioCrawler({ - ...(searchCrawlerOptions as CheerioCrawlerOptions), + ...crawlerOptions, requestQueue: await RequestQueue.open(key, { storageClient: client }), requestHandler: async ({ request, $: _$, addRequests }: CheerioCrawlingContext) => { // NOTE: we need to cast this to fix `cheerio` type errors @@ -135,7 +207,7 @@ export async function createAndStartSearchCrawler( collectedResults: deduplicated, currentPage: nextPage, }, - searchCrawlerOptions.proxyConfiguration, + proxyOptions, nextOffset, ); await addRequests([nextRequest]); @@ -190,9 +262,9 @@ export async function createAndStartContentCrawler( contentCrawlerOptions: ContentCrawlerOptions, startCrawler = true, ) { - const { type: crawlerType, crawlerOptions } = contentCrawlerOptions; + const { type: crawlerType, crawlerOptions, proxyOptions } = contentCrawlerOptions; - const key = getCrawlerKey(crawlerOptions); + const key = getCrawlerKey(crawlerType, crawlerOptions, proxyOptions); if (crawlers.has(key)) { return { key, crawler: crawlers.get(key) }; } @@ -325,18 +397,17 @@ async function maybeCharge(crawlerType: ContentCrawlerTypes, userAuthorization?: } /** - * Adds a search request to the Google search crawler. + * Adds a search request to the Google search crawler identified by `searchCrawlerKey`. * Create a response for the request and set the desired number of results (maxResults). */ export const addSearchRequest = async ( request: RequestOptions, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerKey: string, ) => { - const key = getCrawlerKey(searchCrawlerOptions); - const crawler = crawlers.get(key); + const crawler = crawlers.get(searchCrawlerKey); if (!crawler) { - log.error(`Cheerio crawler not found: key ${key}`); + log.error(`Search crawler not found: key ${searchCrawlerKey}`); return; } addTimeMeasureEvent(request.userData!, 'before-cheerio-queue-add'); diff --git a/src/input.ts b/src/input.ts index cb7597b..1c1b555 100644 --- a/src/input.ts +++ b/src/input.ts @@ -1,6 +1,6 @@ import type { ProxyConfigurationOptions } from 'apify'; import { Actor } from 'apify'; -import type { CheerioCrawlerOptions, ProxyConfiguration } from 'crawlee'; +import type { ProxyConfiguration } from 'crawlee'; import { BrowserName, log } from 'crawlee'; import { firefox } from 'playwright'; @@ -14,8 +14,10 @@ import type { ContentScraperSettings, Input, OutputFormats, + ProxyOptions, RagWebBrowserInput, ScrapingTool, + SearchCrawlerOptions, SERPProxyGroup, UrlToMarkdownInput, } from './types.js'; @@ -59,7 +61,7 @@ async function processInputInternal( ) { const miniActor = getMiniActor(); let input: Input; - let searchCrawlerOptions: CheerioCrawlerOptions = {}; + let searchCrawlerOptions: SearchCrawlerOptions = { crawlerOptions: {}, proxyOptions: {} }; if (miniActor.runsSearch) { const processedRagWebBrowserInput = await processRagWebBrowserInput( @@ -79,8 +81,6 @@ async function processInputInternal( removeCookieWarnings, } = input; - log.setLevel(debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO); - const contentScraperSettings: ContentScraperSettings = { debugMode, dynamicContentWaitSecs, @@ -97,7 +97,7 @@ async function processInputInternal( async function processRagWebBrowserInput(input: Partial, standbyInit: boolean): Promise<{ validatedRagBrowserInput: RagWebBrowserInput; - searchCrawlerOptions: CheerioCrawlerOptions + searchCrawlerOptions: SearchCrawlerOptions }> { /* eslint-disable no-param-reassign */ @@ -166,12 +166,16 @@ async function processRagWebBrowserInput(input: Partial, sta input.dynamicContentWaitSecs = Math.round(input.requestTimeoutSecs / 2); } - const proxySearch = await Actor.createProxyConfiguration({ groups: [input.serpProxyGroup], checkAccess: false }); - const searchCrawlerOptions: CheerioCrawlerOptions = { - keepAlive: standbyInit, - maxRequestRetries: input.serpMaxRetries, - proxyConfiguration: proxySearch, - autoscaledPoolOptions: { desiredConcurrency: 1 }, + const proxyOptions: ProxyOptions = { groups: [input.serpProxyGroup] }; + const proxySearch = await Actor.createProxyConfiguration({ ...proxyOptions, checkAccess: false }); + const searchCrawlerOptions: SearchCrawlerOptions = { + crawlerOptions: { + keepAlive: standbyInit, + maxRequestRetries: input.serpMaxRetries, + proxyConfiguration: proxySearch, + autoscaledPoolOptions: { desiredConcurrency: 1 }, + }, + proxyOptions, }; const validatedRagBrowserInput = validateAndFillInput(input) as RagWebBrowserInput; return { @@ -219,6 +223,7 @@ function createPlaywrightCrawlerOptions( return { type: ContentCrawlerTypes.PLAYWRIGHT, + proxyOptions: input.proxyConfiguration, crawlerOptions: { headless: true, keepAlive, @@ -261,6 +266,7 @@ function createCheerioCrawlerOptions( return { type: ContentCrawlerTypes.CHEERIO, + proxyOptions: input.proxyConfiguration, crawlerOptions: { keepAlive, maxRequestRetries, diff --git a/src/main.ts b/src/main.ts index 402cf4a..c36632d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -18,6 +18,11 @@ Actor.on('migrating', () => { const originalInput = await Actor.getInput>() ?? {} as Input; +// Set once, from the run input: the level is process-wide, so letting a standby request change it +// would leak debug output across concurrent callers. A request's own `debugMode` still controls the +// `debug` field of its response, see `ContentScraperSettings`. +log.setLevel(originalInput.debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO); + if (isActorStandby()) { log.info('Actor is running in the STANDBY mode.'); diff --git a/src/search.ts b/src/search.ts index 620c460..594bd98 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,6 +1,6 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; -import { type CheerioCrawlerOptions, log } from 'crawlee'; +import { log } from 'crawlee'; import { PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS } from './const.js'; import { addContentCrawlRequest, addSearchRequest, createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js'; @@ -8,7 +8,15 @@ import { UserInputError } from './errors.js'; import { processInput } from './input.js'; import { getMiniActor } from './mini-actors.js'; import { createResponsePromise } from './responses.js'; -import type { ContentCrawlerOptions, ContentScraperSettings, Input, Output, RagWebBrowserInput, UrlToMarkdownInput } from './types.js'; +import type { + ContentCrawlerOptions, + ContentScraperSettings, + Input, + Output, + RagWebBrowserInput, + SearchCrawlerOptions, + UrlToMarkdownInput, +} from './types.js'; import { addTimeMeasureEvent, createRequest, @@ -26,7 +34,7 @@ import { */ function prepareRequest( input: Input, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, contentCrawlerKey: string, contentScraperSettings: ContentScraperSettings, userAuthorization?: string, @@ -80,7 +88,7 @@ function prepareRequest( contentScraperSettings, userAuthorization, }, - searchCrawlerOptions.proxyConfiguration, + searchCrawlerOptions.proxyOptions, ); addTimeMeasureEvent(req.userData!, 'request-received', Date.now()); @@ -101,7 +109,7 @@ async function runSearchProcess(params: Partial, userAuthorization?: stri } = await processInput(params); // Set keepAlive to true to find the correct crawlers - searchCrawlerOptions.keepAlive = true; + searchCrawlerOptions.crawlerOptions.keepAlive = true; contentCrawlerOptions.crawlerOptions.keepAlive = true; const { key: contentCrawlerKey } = await createAndStartContentCrawler(contentCrawlerOptions); @@ -125,9 +133,9 @@ async function runSearchProcess(params: Partial, userAuthorization?: stri } await addContentCrawlRequest(req, responseId, contentCrawlerKey); } else { - await createAndStartSearchCrawler(searchCrawlerOptions); // If input is a search query, run the search crawler first - await addSearchRequest(req, searchCrawlerOptions); + const { key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions); + await addSearchRequest(req, searchCrawlerKey); } // Return promise that resolves when all requests are processed @@ -178,7 +186,7 @@ export async function handleModelContextProtocol(params: Partial, userAut */ export async function handleSearchNormalMode( input: Input, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, contentCrawlerOptions: ContentCrawlerOptions, contentScraperSettings: ContentScraperSettings, ) { @@ -205,8 +213,8 @@ export async function handleSearchNormalMode( } await addContentCrawlRequest(req, '', contentCrawlerKey); } else { - const { crawler: searchCrawler } = await createAndStartSearchCrawler(searchCrawlerOptions, false); - await addSearchRequest(req, searchCrawlerOptions); + const { crawler: searchCrawler, key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions, false); + await addSearchRequest(req, searchCrawlerKey); addTimeMeasureEvent(req.userData!, 'before-cheerio-run', startedTime); log.info(`Running Google Search crawler with request: ${JSON.stringify(req)}`); await searchCrawler!.run(); diff --git a/src/types.ts b/src/types.ts index d5c5863..68f7f1f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,6 +12,9 @@ import type { ContentCrawlerTypes } from './const.js'; */ type Optional = Omit & Partial>; +/** What `Actor.createProxyConfiguration` accepts: the SDK options plus the input-schema flag. */ +export type ProxyOptions = ProxyConfigurationOptions & { useApifyProxy?: boolean }; + export type OutputFormats = 'text' | 'markdown' | 'html'; export type SERPProxyGroup = 'GOOGLE_SERP' | 'SHADER'; export type ScrapingTool = 'browser-playwright' | 'raw-http'; @@ -159,10 +162,22 @@ export type Output = { }; }; +/** + * Crawler settings, paired with the proxy options they were built from. Keeping the options is what + * lets `getCrawlerKey` and the SERP protocol choice avoid reading the constructed + * `ProxyConfiguration`, whose serialization is neither stable nor free of secrets. + */ +export type SearchCrawlerOptions = { + crawlerOptions: CheerioCrawlerOptions; + proxyOptions: ProxyOptions; +}; + export type ContentCrawlerOptions = { type: ContentCrawlerTypes.CHEERIO, - crawlerOptions: CheerioCrawlerOptions + crawlerOptions: CheerioCrawlerOptions, + proxyOptions: ProxyOptions, } | { type: ContentCrawlerTypes.PLAYWRIGHT, - crawlerOptions: PlaywrightCrawlerOptions + crawlerOptions: PlaywrightCrawlerOptions, + proxyOptions: ProxyOptions, }; diff --git a/src/utils.ts b/src/utils.ts index 19e2892..94d8a19 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,7 @@ import type { IncomingHttpHeaders } from 'node:http'; import { parse } from 'node:querystring'; import { Actor } from 'apify'; -import type { ProxyConfiguration, RequestOptions } from 'crawlee'; +import type { RequestOptions } from 'crawlee'; import { log } from 'crawlee'; import ragWebBrowserInputSchema from '../actors/apify_rag-web-browser/.actor/input_schema.json' with { type: 'json' }; @@ -14,6 +14,7 @@ import type { CreateSearchRequestUserData, Input, OrganicResult, OutputFormats, + ProxyOptions, SearchCrawlerUserData, TimeMeasure, } from './types.js'; @@ -128,7 +129,7 @@ export function randomId() { */ export function createSearchRequest( userData: CreateSearchRequestUserData, - proxyConfiguration: ProxyConfiguration | undefined, + proxyOptions: ProxyOptions, startOffset = 0, ): RequestOptions { // Initialize or update pagination fields @@ -136,9 +137,8 @@ export function createSearchRequest( const currentPage = userData.currentPage ?? 0; const totalPages = userData.totalPages ?? Math.ceil(userData.maxResults / 10) + 1; - // @ts-expect-error is there a better way to get group information? - // (e.g. to create extended CheerioCrawlOptions and pass it there?) - const groups = proxyConfiguration?.groups || []; + // Apify's Google SERP proxy only allows plain HTTP requests. + const groups = proxyOptions.groups ?? []; const protocol = groups.includes('GOOGLE_SERP') ? 'http' : 'https'; const urlSearch = startOffset > 0 ? `${protocol}://www.google.com/search?q=${encodeURI(userData.query)}&start=${startOffset}` diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts index ec615d6..ffc091f 100644 --- a/tests/cheerio-crawler.content.test.ts +++ b/tests/cheerio-crawler.content.test.ts @@ -88,7 +88,7 @@ describe('Cheerio Crawler Content Tests', () => { it('test the crawler is created with the impit HTTP client', async () => { const { crawler } = await createAndStartContentCrawler( - { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {} }, + { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {}, proxyOptions: {} }, false, ); diff --git a/tests/crawler-key.test.ts b/tests/crawler-key.test.ts new file mode 100644 index 0000000..fcf074f --- /dev/null +++ b/tests/crawler-key.test.ts @@ -0,0 +1,128 @@ +import type { CheerioCrawlerOptions } from 'crawlee'; +import { log } from 'crawlee'; +import { beforeAll, describe, expect, it } from 'vitest'; + +import { ContentCrawlerTypes } from '../src/const.js'; +import { getCrawlerKey } from '../src/crawlers.js'; +import { processInput, processStandbyInput } from '../src/input.js'; +import type { ProxyOptions } from '../src/types.js'; +import { parseParameters } from '../src/utils.js'; + +const baseOptions: CheerioCrawlerOptions = { + keepAlive: true, + maxRequestRetries: 1, + requestHandlerTimeoutSecs: 40, + autoscaledPoolOptions: { desiredConcurrency: 5 }, +}; + +const cheerioKey = ( + options: CheerioCrawlerOptions = {}, + proxyOptions: ProxyOptions = { useApifyProxy: true }, +) => getCrawlerKey(ContentCrawlerTypes.CHEERIO, { ...baseOptions, ...options }, proxyOptions); + +describe('getCrawlerKey', () => { + // The key doubles as a request queue name, so it has to stay a short slug. + it('is a slug of the crawler kind and a hash', () => { + expect(cheerioKey()).toMatch(/^cheerio-[0-9a-f]+$/); + }); + + // The three crawlers can otherwise share a fingerprint, so the kind carries the distinction. + it('separates the crawler kinds', () => { + const keys = new Set([ + getCrawlerKey('search', baseOptions, {}), + getCrawlerKey(ContentCrawlerTypes.CHEERIO, baseOptions, {}), + getCrawlerKey(ContentCrawlerTypes.PLAYWRIGHT, baseOptions, {}), + ]); + + expect(keys.size).toBe(3); + }); + + it('ignores the order the proxy options were declared in', () => { + const a = cheerioKey({}, { useApifyProxy: true, countryCode: 'US' }); + const b = cheerioKey({}, { countryCode: 'US', useApifyProxy: true }); + + expect(a).toBe(b); + }); + + it('treats the apifyProxy* input-schema aliases as their canonical counterparts', () => { + expect(cheerioKey({}, { apifyProxyGroups: ['RESIDENTIAL'] })).toBe(cheerioKey({}, { groups: ['RESIDENTIAL'] })); + expect(cheerioKey({}, { apifyProxyCountry: 'US' })).toBe(cheerioKey({}, { countryCode: 'US' })); + expect(cheerioKey({}, { useApifyProxy: true })).toBe(cheerioKey({}, {})); + }); + + it('never exposes proxy credentials, so the key is safe to log', () => { + const key = cheerioKey({}, { + password: 'hunter2', + proxyUrls: ['http://user:hunter2@proxy.example.com:8000'], + }); + + expect(key).not.toContain('hunter2'); + expect(key).not.toContain('proxy.example.com'); + }); + + it('still separates crawlers whose settings genuinely differ', () => { + const keys = new Set([ + cheerioKey(), + cheerioKey({ keepAlive: false }), + cheerioKey({ maxRequestRetries: 3 }), + cheerioKey({ requestHandlerTimeoutSecs: 90 }), + cheerioKey({ autoscaledPoolOptions: { desiredConcurrency: 10 } }), + cheerioKey({}, { groups: ['RESIDENTIAL'] }), + cheerioKey({}, { countryCode: 'US' }), + cheerioKey({}, { useApifyProxy: false }), + // A caller supplying its own proxy password must not land on the run's own crawler. + cheerioKey({}, { password: 'hunter2' }), + cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }), + cheerioKey({}, { proxyUrls: ['http://other.example.com:8000'] }), + ]); + + expect(keys.size).toBe(11); + }); +}); + +// Regression test for https://github.com/apify/actor-rag-web-browser/issues/60. +describe('standby requests reuse the crawlers started at boot', () => { + process.env.ACTOR_FULL_NAME = 'apify/rag-web-browser'; + + // A custom proxy keeps `Actor.createProxyConfiguration` off the network, since it only checks + // access for Apify Proxy. Without one it returns `undefined` unless `APIFY_PROXY_PASSWORD` is + // set, and the content crawlers would then be keyed without a `ProxyConfiguration` at all. + const proxyConfiguration = { useApifyProxy: false, proxyUrls: ['http://proxy.invalid:8000'] }; + const query = (extraParams = '') => `?query=hello&proxyConfiguration=${ + encodeURIComponent(JSON.stringify(proxyConfiguration))}${extraParams}`; + + let bootKeys: string[]; + + const keysForRequest = async (queryString: string) => { + const { searchCrawlerOptions, contentCrawlerOptions } = await processInput(parseParameters(queryString)); + // Mirrors `runSearchProcess`, which forces keepAlive to match the crawlers started at boot. + searchCrawlerOptions.crawlerOptions.keepAlive = true; + contentCrawlerOptions.crawlerOptions.keepAlive = true; + + return [ + getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), + getCrawlerKey(contentCrawlerOptions.type, contentCrawlerOptions.crawlerOptions, contentCrawlerOptions.proxyOptions), + ]; + }; + + beforeAll(async () => { + const { searchCrawlerOptions, contentCrawlerOptions } = await processStandbyInput({ proxyConfiguration }); + + bootKeys = [ + getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), + ...contentCrawlerOptions.map((o) => getCrawlerKey(o.type, o.crawlerOptions, o.proxyOptions)), + ]; + expect(new Set(bootKeys).size).toBe(3); + }); + + it('reuses them for a plain request', async () => { + expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query()))); + }); + + it('reuses them when debugMode is requested, and leaves the log level alone', async () => { + const levelBefore = log.getLevel(); + + expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query('&debugMode=true')))); + expect(log.getLevel()).toBe(levelBefore); + }); +}); From 4061ca5980932be9c4dbdce67feb370bba71eeb8 Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Thu, 6 Aug 2026 16:52:19 +0200 Subject: [PATCH 2/2] fix: collapse equivalent proxy option spellings in the crawler key --- src/crawlers.ts | 49 +++++++++++++++++---------------------- src/main.ts | 5 ++-- src/types.ts | 6 +---- src/utils.ts | 1 - tests/crawler-key.test.ts | 17 +++++++++----- 5 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/crawlers.ts b/src/crawlers.ts index 581e225..b0c8ccc 100644 --- a/src/crawlers.ts +++ b/src/crawlers.ts @@ -58,33 +58,31 @@ async function getGhosteryBlocker(): Promise { } } -/** `checkAccess` only drives initialization, and no serialization tells two functions apart. */ -const PROXY_OPTIONS_EXCLUDED_FROM_KEY = new Set(['checkAccess', 'newUrlFunction']); - -/** - * Resolves the aliases that `ProxyConfiguration` itself resolves, so options that differ only in - * which spelling they use share one crawler instead of getting one each. - */ +/** Mirrors how `Actor.createProxyConfiguration` reads these options, so equivalent spellings share a crawler. */ function resolveProxyOptions(proxyOptions: ProxyOptions) { - const { apifyProxyGroups, apifyProxyCountry, apifyProxySubdivision, ...rest } = proxyOptions; - const resolved = { + const { + useApifyProxy, + checkAccess, + newUrlFunction, + apifyProxyGroups, + apifyProxyCountry, + apifyProxySubdivision, + ...rest + } = proxyOptions; + + if (useApifyProxy === false && !rest.proxyUrls) { + return null; + } + + return { ...rest, - useApifyProxy: rest.useApifyProxy !== false, groups: rest.groups?.length ? rest.groups : apifyProxyGroups, countryCode: rest.countryCode || apifyProxyCountry, subdivisionCode: rest.subdivisionCode || apifyProxySubdivision, }; - - return Object.fromEntries( - Object.entries(resolved).filter(([key]) => !PROXY_OPTIONS_EXCLUDED_FROM_KEY.has(key)), - ); } -/** - * `JSON.stringify` with object keys sorted at every level, so that two objects differing only in key - * order serialize identically. Array order is kept because it carries meaning, such as the rotation - * order of `proxyUrls`. `Date` and `toJSON` are not honoured; the input here is parsed JSON. - */ +/** `JSON.stringify` with object keys sorted at every level. Array order is kept, it carries meaning. */ function canonicalJson(value: unknown): string { if (value === null || typeof value !== 'object') { return JSON.stringify(value) ?? 'null'; @@ -99,15 +97,10 @@ function canonicalJson(value: unknown): string { } /** - * Identifies a crawler in the `crawlers` cache. The listed fields are the only ones the option - * builders in `input.ts` derive from the input; everything else they set is a constant. - * - * The proxy *options* stand in for the constructed `ProxyConfiguration`, whose child logger - * snapshots the log level at construction time. Serializing that instance made identical requests - * miss the cache: https://github.com/apify/actor-rag-web-browser/issues/60. - * - * The fingerprint is hashed because the key is logged and used as the request queue name, while the - * options may hold the proxy password or a custom proxy URL with credentials in it. + * Identifies a crawler in the `crawlers` cache. Listed are the only options the builders in + * `input.ts` derive from the input; the proxy options stand in for the constructed + * `ProxyConfiguration`, whose child logger snapshots the log level and so kept changing the key. + * Hashed because the key is logged and used as a queue name, while the options can hold credentials. */ export function getCrawlerKey( kind: CrawlerKind, diff --git a/src/main.ts b/src/main.ts index c36632d..0dad7cf 100644 --- a/src/main.ts +++ b/src/main.ts @@ -18,9 +18,8 @@ Actor.on('migrating', () => { const originalInput = await Actor.getInput>() ?? {} as Input; -// Set once, from the run input: the level is process-wide, so letting a standby request change it -// would leak debug output across concurrent callers. A request's own `debugMode` still controls the -// `debug` field of its response, see `ContentScraperSettings`. +// Set once: the level is process-wide, so a standby request changing it would leak debug output +// across concurrent callers. Per-request `debugMode` still fills the response's `debug` field. log.setLevel(originalInput.debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO); if (isActorStandby()) { diff --git a/src/types.ts b/src/types.ts index 68f7f1f..5b379d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -162,11 +162,7 @@ export type Output = { }; }; -/** - * Crawler settings, paired with the proxy options they were built from. Keeping the options is what - * lets `getCrawlerKey` and the SERP protocol choice avoid reading the constructed - * `ProxyConfiguration`, whose serialization is neither stable nor free of secrets. - */ +/** The proxy options are kept so nothing has to read them back off the constructed `ProxyConfiguration`. */ export type SearchCrawlerOptions = { crawlerOptions: CheerioCrawlerOptions; proxyOptions: ProxyOptions; diff --git a/src/utils.ts b/src/utils.ts index 94d8a19..e163061 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -137,7 +137,6 @@ export function createSearchRequest( const currentPage = userData.currentPage ?? 0; const totalPages = userData.totalPages ?? Math.ceil(userData.maxResults / 10) + 1; - // Apify's Google SERP proxy only allows plain HTTP requests. const groups = proxyOptions.groups ?? []; const protocol = groups.includes('GOOGLE_SERP') ? 'http' : 'https'; const urlSearch = startOffset > 0 diff --git a/tests/crawler-key.test.ts b/tests/crawler-key.test.ts index fcf074f..7a84df3 100644 --- a/tests/crawler-key.test.ts +++ b/tests/crawler-key.test.ts @@ -26,7 +26,6 @@ describe('getCrawlerKey', () => { expect(cheerioKey()).toMatch(/^cheerio-[0-9a-f]+$/); }); - // The three crawlers can otherwise share a fingerprint, so the kind carries the distinction. it('separates the crawler kinds', () => { const keys = new Set([ getCrawlerKey('search', baseOptions, {}), @@ -47,7 +46,16 @@ describe('getCrawlerKey', () => { it('treats the apifyProxy* input-schema aliases as their canonical counterparts', () => { expect(cheerioKey({}, { apifyProxyGroups: ['RESIDENTIAL'] })).toBe(cheerioKey({}, { groups: ['RESIDENTIAL'] })); expect(cheerioKey({}, { apifyProxyCountry: 'US' })).toBe(cheerioKey({}, { countryCode: 'US' })); + }); + + it('collapses the useApifyProxy spellings the way the SDK does', () => { + const noProxy = cheerioKey({}, { useApifyProxy: false }); + const custom = cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }); + expect(cheerioKey({}, { useApifyProxy: true })).toBe(cheerioKey({}, {})); + expect(cheerioKey({}, { useApifyProxy: false, proxyUrls: ['http://proxy.example.com:8000'] })).toBe(custom); + expect(cheerioKey({}, { useApifyProxy: false, tieredProxyUrls: [['http://a:1']] })).toBe(noProxy); + expect(noProxy).not.toBe(custom); }); it('never exposes proxy credentials, so the key is safe to log', () => { @@ -70,7 +78,6 @@ describe('getCrawlerKey', () => { cheerioKey({}, { groups: ['RESIDENTIAL'] }), cheerioKey({}, { countryCode: 'US' }), cheerioKey({}, { useApifyProxy: false }), - // A caller supplying its own proxy password must not land on the run's own crawler. cheerioKey({}, { password: 'hunter2' }), cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }), cheerioKey({}, { proxyUrls: ['http://other.example.com:8000'] }), @@ -80,13 +87,11 @@ describe('getCrawlerKey', () => { }); }); -// Regression test for https://github.com/apify/actor-rag-web-browser/issues/60. describe('standby requests reuse the crawlers started at boot', () => { process.env.ACTOR_FULL_NAME = 'apify/rag-web-browser'; - // A custom proxy keeps `Actor.createProxyConfiguration` off the network, since it only checks - // access for Apify Proxy. Without one it returns `undefined` unless `APIFY_PROXY_PASSWORD` is - // set, and the content crawlers would then be keyed without a `ProxyConfiguration` at all. + // A custom proxy keeps `Actor.createProxyConfiguration` off the network and off + // `APIFY_PROXY_PASSWORD`, which it needs to return a `ProxyConfiguration` at all. const proxyConfiguration = { useApifyProxy: false, proxyUrls: ['http://proxy.invalid:8000'] }; const query = (extraParams = '') => `?query=hello&proxyConfiguration=${ encodeURIComponent(JSON.stringify(proxyConfiguration))}${extraParams}`;