diff --git a/src/const.ts b/src/const.ts index f8baaa0..7c98f83 100644 --- a/src/const.ts +++ b/src/const.ts @@ -18,6 +18,8 @@ export enum ContentCrawlerTypes { CHEERIO = 'cheerio', } +export type CrawlerKind = 'search' | ContentCrawlerTypes; + export const PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS = 60; export const GOOGLE_STANDARD_RESULTS_PER_PAGE = 10; diff --git a/src/crawlers.ts b/src/crawlers.ts index 9515bd3..b0c8ccc 100644 --- a/src/crawlers.ts +++ b/src/crawlers.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFile } from 'node:fs/promises'; import { ImpitHttpClient } from '@crawlee/impit-client'; @@ -16,12 +17,19 @@ import { type RequestOptions, } from 'crawlee'; +import type { CrawlerKind } from './const.js'; import { ContentCrawlerTypes, GOOGLE_STANDARD_RESULTS_PER_PAGE } from './const.js'; import { deduplicateResults, scrapeOrganicResults } from './google-search/google-extractors-urls.js'; import { getMiniActor } from './mini-actors.js'; import { failedRequestHandler, requestHandlerCheerio, requestHandlerPlaywright } from './request-handler.js'; import { addEmptyResultToResponse, sendResponseError } from './responses.js'; -import type { ContentCrawlerOptions, ContentCrawlerUserData, SearchCrawlerUserData } from './types.js'; +import type { + ContentCrawlerOptions, + ContentCrawlerUserData, + ProxyOptions, + SearchCrawlerOptions, + SearchCrawlerUserData, +} from './types.js'; import { addTimeMeasureEvent, createRequest, createSearchRequest, isActorStandby, randomId } from './utils.js'; const crawlers = new Map(); @@ -50,8 +58,64 @@ async function getGhosteryBlocker(): Promise { } } -export function getCrawlerKey(crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions) { - return JSON.stringify(crawlerOptions); +/** Mirrors how `Actor.createProxyConfiguration` reads these options, so equivalent spellings share a crawler. */ +function resolveProxyOptions(proxyOptions: ProxyOptions) { + const { + useApifyProxy, + checkAccess, + newUrlFunction, + apifyProxyGroups, + apifyProxyCountry, + apifyProxySubdivision, + ...rest + } = proxyOptions; + + if (useApifyProxy === false && !rest.proxyUrls) { + return null; + } + + return { + ...rest, + groups: rest.groups?.length ? rest.groups : apifyProxyGroups, + countryCode: rest.countryCode || apifyProxyCountry, + subdivisionCode: rest.subdivisionCode || apifyProxySubdivision, + }; +} + +/** `JSON.stringify` with object keys sorted at every level. Array order is kept, it carries meaning. */ +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value) ?? 'null'; + } + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + const entries = Object.entries(value) + .filter(([, entryValue]) => entryValue !== undefined) + .sort(([a], [b]) => (a < b ? -1 : 1)); + return `{${entries.map(([key, entryValue]) => `${JSON.stringify(key)}:${canonicalJson(entryValue)}`).join(',')}}`; +} + +/** + * Identifies a crawler in the `crawlers` cache. Listed are the only options the builders in + * `input.ts` derive from the input; the proxy options stand in for the constructed + * `ProxyConfiguration`, whose child logger snapshots the log level and so kept changing the key. + * Hashed because the key is logged and used as a queue name, while the options can hold credentials. + */ +export function getCrawlerKey( + kind: CrawlerKind, + crawlerOptions: CheerioCrawlerOptions | PlaywrightCrawlerOptions, + proxyOptions: ProxyOptions, +): string { + const fingerprint = { + keepAlive: crawlerOptions.keepAlive, + maxRequestRetries: crawlerOptions.maxRequestRetries, + requestHandlerTimeoutSecs: crawlerOptions.requestHandlerTimeoutSecs, + desiredConcurrency: crawlerOptions.autoscaledPoolOptions?.desiredConcurrency, + proxy: resolveProxyOptions(proxyOptions), + }; + const hash = createHash('sha1').update(canonicalJson(fingerprint)).digest('hex'); + return `${kind}-${hash.slice(0, 12)}`; } /** @@ -86,17 +150,18 @@ export const addContentCrawlRequest = async ( * A crawler won't be created if it already exists. */ export async function createAndStartSearchCrawler( - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, startCrawler = true, ) { - const key = getCrawlerKey(searchCrawlerOptions); + const { crawlerOptions, proxyOptions } = searchCrawlerOptions; + const key = getCrawlerKey('search', crawlerOptions, proxyOptions); if (crawlers.has(key)) { return { key, crawler: crawlers.get(key) }; } log.info(`Creating new cheerio crawler with key ${key}`); const crawler = new CheerioCrawler({ - ...(searchCrawlerOptions as CheerioCrawlerOptions), + ...crawlerOptions, requestQueue: await RequestQueue.open(key, { storageClient: client }), requestHandler: async ({ request, $: _$, addRequests }: CheerioCrawlingContext) => { // NOTE: we need to cast this to fix `cheerio` type errors @@ -135,7 +200,7 @@ export async function createAndStartSearchCrawler( collectedResults: deduplicated, currentPage: nextPage, }, - searchCrawlerOptions.proxyConfiguration, + proxyOptions, nextOffset, ); await addRequests([nextRequest]); @@ -190,9 +255,9 @@ export async function createAndStartContentCrawler( contentCrawlerOptions: ContentCrawlerOptions, startCrawler = true, ) { - const { type: crawlerType, crawlerOptions } = contentCrawlerOptions; + const { type: crawlerType, crawlerOptions, proxyOptions } = contentCrawlerOptions; - const key = getCrawlerKey(crawlerOptions); + const key = getCrawlerKey(crawlerType, crawlerOptions, proxyOptions); if (crawlers.has(key)) { return { key, crawler: crawlers.get(key) }; } @@ -325,18 +390,17 @@ async function maybeCharge(crawlerType: ContentCrawlerTypes, userAuthorization?: } /** - * Adds a search request to the Google search crawler. + * Adds a search request to the Google search crawler identified by `searchCrawlerKey`. * Create a response for the request and set the desired number of results (maxResults). */ export const addSearchRequest = async ( request: RequestOptions, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerKey: string, ) => { - const key = getCrawlerKey(searchCrawlerOptions); - const crawler = crawlers.get(key); + const crawler = crawlers.get(searchCrawlerKey); if (!crawler) { - log.error(`Cheerio crawler not found: key ${key}`); + log.error(`Search crawler not found: key ${searchCrawlerKey}`); return; } addTimeMeasureEvent(request.userData!, 'before-cheerio-queue-add'); diff --git a/src/input.ts b/src/input.ts index cb7597b..1c1b555 100644 --- a/src/input.ts +++ b/src/input.ts @@ -1,6 +1,6 @@ import type { ProxyConfigurationOptions } from 'apify'; import { Actor } from 'apify'; -import type { CheerioCrawlerOptions, ProxyConfiguration } from 'crawlee'; +import type { ProxyConfiguration } from 'crawlee'; import { BrowserName, log } from 'crawlee'; import { firefox } from 'playwright'; @@ -14,8 +14,10 @@ import type { ContentScraperSettings, Input, OutputFormats, + ProxyOptions, RagWebBrowserInput, ScrapingTool, + SearchCrawlerOptions, SERPProxyGroup, UrlToMarkdownInput, } from './types.js'; @@ -59,7 +61,7 @@ async function processInputInternal( ) { const miniActor = getMiniActor(); let input: Input; - let searchCrawlerOptions: CheerioCrawlerOptions = {}; + let searchCrawlerOptions: SearchCrawlerOptions = { crawlerOptions: {}, proxyOptions: {} }; if (miniActor.runsSearch) { const processedRagWebBrowserInput = await processRagWebBrowserInput( @@ -79,8 +81,6 @@ async function processInputInternal( removeCookieWarnings, } = input; - log.setLevel(debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO); - const contentScraperSettings: ContentScraperSettings = { debugMode, dynamicContentWaitSecs, @@ -97,7 +97,7 @@ async function processInputInternal( async function processRagWebBrowserInput(input: Partial, standbyInit: boolean): Promise<{ validatedRagBrowserInput: RagWebBrowserInput; - searchCrawlerOptions: CheerioCrawlerOptions + searchCrawlerOptions: SearchCrawlerOptions }> { /* eslint-disable no-param-reassign */ @@ -166,12 +166,16 @@ async function processRagWebBrowserInput(input: Partial, sta input.dynamicContentWaitSecs = Math.round(input.requestTimeoutSecs / 2); } - const proxySearch = await Actor.createProxyConfiguration({ groups: [input.serpProxyGroup], checkAccess: false }); - const searchCrawlerOptions: CheerioCrawlerOptions = { - keepAlive: standbyInit, - maxRequestRetries: input.serpMaxRetries, - proxyConfiguration: proxySearch, - autoscaledPoolOptions: { desiredConcurrency: 1 }, + const proxyOptions: ProxyOptions = { groups: [input.serpProxyGroup] }; + const proxySearch = await Actor.createProxyConfiguration({ ...proxyOptions, checkAccess: false }); + const searchCrawlerOptions: SearchCrawlerOptions = { + crawlerOptions: { + keepAlive: standbyInit, + maxRequestRetries: input.serpMaxRetries, + proxyConfiguration: proxySearch, + autoscaledPoolOptions: { desiredConcurrency: 1 }, + }, + proxyOptions, }; const validatedRagBrowserInput = validateAndFillInput(input) as RagWebBrowserInput; return { @@ -219,6 +223,7 @@ function createPlaywrightCrawlerOptions( return { type: ContentCrawlerTypes.PLAYWRIGHT, + proxyOptions: input.proxyConfiguration, crawlerOptions: { headless: true, keepAlive, @@ -261,6 +266,7 @@ function createCheerioCrawlerOptions( return { type: ContentCrawlerTypes.CHEERIO, + proxyOptions: input.proxyConfiguration, crawlerOptions: { keepAlive, maxRequestRetries, diff --git a/src/main.ts b/src/main.ts index 402cf4a..0dad7cf 100644 --- a/src/main.ts +++ b/src/main.ts @@ -18,6 +18,10 @@ Actor.on('migrating', () => { const originalInput = await Actor.getInput>() ?? {} as Input; +// Set once: the level is process-wide, so a standby request changing it would leak debug output +// across concurrent callers. Per-request `debugMode` still fills the response's `debug` field. +log.setLevel(originalInput.debugMode ? log.LEVELS.DEBUG : log.LEVELS.INFO); + if (isActorStandby()) { log.info('Actor is running in the STANDBY mode.'); diff --git a/src/search.ts b/src/search.ts index 620c460..594bd98 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,6 +1,6 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; -import { type CheerioCrawlerOptions, log } from 'crawlee'; +import { log } from 'crawlee'; import { PLAYWRIGHT_REQUEST_TIMEOUT_NORMAL_MODE_SECS } from './const.js'; import { addContentCrawlRequest, addSearchRequest, createAndStartContentCrawler, createAndStartSearchCrawler } from './crawlers.js'; @@ -8,7 +8,15 @@ import { UserInputError } from './errors.js'; import { processInput } from './input.js'; import { getMiniActor } from './mini-actors.js'; import { createResponsePromise } from './responses.js'; -import type { ContentCrawlerOptions, ContentScraperSettings, Input, Output, RagWebBrowserInput, UrlToMarkdownInput } from './types.js'; +import type { + ContentCrawlerOptions, + ContentScraperSettings, + Input, + Output, + RagWebBrowserInput, + SearchCrawlerOptions, + UrlToMarkdownInput, +} from './types.js'; import { addTimeMeasureEvent, createRequest, @@ -26,7 +34,7 @@ import { */ function prepareRequest( input: Input, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, contentCrawlerKey: string, contentScraperSettings: ContentScraperSettings, userAuthorization?: string, @@ -80,7 +88,7 @@ function prepareRequest( contentScraperSettings, userAuthorization, }, - searchCrawlerOptions.proxyConfiguration, + searchCrawlerOptions.proxyOptions, ); addTimeMeasureEvent(req.userData!, 'request-received', Date.now()); @@ -101,7 +109,7 @@ async function runSearchProcess(params: Partial, userAuthorization?: stri } = await processInput(params); // Set keepAlive to true to find the correct crawlers - searchCrawlerOptions.keepAlive = true; + searchCrawlerOptions.crawlerOptions.keepAlive = true; contentCrawlerOptions.crawlerOptions.keepAlive = true; const { key: contentCrawlerKey } = await createAndStartContentCrawler(contentCrawlerOptions); @@ -125,9 +133,9 @@ async function runSearchProcess(params: Partial, userAuthorization?: stri } await addContentCrawlRequest(req, responseId, contentCrawlerKey); } else { - await createAndStartSearchCrawler(searchCrawlerOptions); // If input is a search query, run the search crawler first - await addSearchRequest(req, searchCrawlerOptions); + const { key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions); + await addSearchRequest(req, searchCrawlerKey); } // Return promise that resolves when all requests are processed @@ -178,7 +186,7 @@ export async function handleModelContextProtocol(params: Partial, userAut */ export async function handleSearchNormalMode( input: Input, - searchCrawlerOptions: CheerioCrawlerOptions, + searchCrawlerOptions: SearchCrawlerOptions, contentCrawlerOptions: ContentCrawlerOptions, contentScraperSettings: ContentScraperSettings, ) { @@ -205,8 +213,8 @@ export async function handleSearchNormalMode( } await addContentCrawlRequest(req, '', contentCrawlerKey); } else { - const { crawler: searchCrawler } = await createAndStartSearchCrawler(searchCrawlerOptions, false); - await addSearchRequest(req, searchCrawlerOptions); + const { crawler: searchCrawler, key: searchCrawlerKey } = await createAndStartSearchCrawler(searchCrawlerOptions, false); + await addSearchRequest(req, searchCrawlerKey); addTimeMeasureEvent(req.userData!, 'before-cheerio-run', startedTime); log.info(`Running Google Search crawler with request: ${JSON.stringify(req)}`); await searchCrawler!.run(); diff --git a/src/types.ts b/src/types.ts index d5c5863..5b379d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,6 +12,9 @@ import type { ContentCrawlerTypes } from './const.js'; */ type Optional = Omit & Partial>; +/** What `Actor.createProxyConfiguration` accepts: the SDK options plus the input-schema flag. */ +export type ProxyOptions = ProxyConfigurationOptions & { useApifyProxy?: boolean }; + export type OutputFormats = 'text' | 'markdown' | 'html'; export type SERPProxyGroup = 'GOOGLE_SERP' | 'SHADER'; export type ScrapingTool = 'browser-playwright' | 'raw-http'; @@ -159,10 +162,18 @@ export type Output = { }; }; +/** The proxy options are kept so nothing has to read them back off the constructed `ProxyConfiguration`. */ +export type SearchCrawlerOptions = { + crawlerOptions: CheerioCrawlerOptions; + proxyOptions: ProxyOptions; +}; + export type ContentCrawlerOptions = { type: ContentCrawlerTypes.CHEERIO, - crawlerOptions: CheerioCrawlerOptions + crawlerOptions: CheerioCrawlerOptions, + proxyOptions: ProxyOptions, } | { type: ContentCrawlerTypes.PLAYWRIGHT, - crawlerOptions: PlaywrightCrawlerOptions + crawlerOptions: PlaywrightCrawlerOptions, + proxyOptions: ProxyOptions, }; diff --git a/src/utils.ts b/src/utils.ts index 19e2892..e163061 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,7 @@ import type { IncomingHttpHeaders } from 'node:http'; import { parse } from 'node:querystring'; import { Actor } from 'apify'; -import type { ProxyConfiguration, RequestOptions } from 'crawlee'; +import type { RequestOptions } from 'crawlee'; import { log } from 'crawlee'; import ragWebBrowserInputSchema from '../actors/apify_rag-web-browser/.actor/input_schema.json' with { type: 'json' }; @@ -14,6 +14,7 @@ import type { CreateSearchRequestUserData, Input, OrganicResult, OutputFormats, + ProxyOptions, SearchCrawlerUserData, TimeMeasure, } from './types.js'; @@ -128,7 +129,7 @@ export function randomId() { */ export function createSearchRequest( userData: CreateSearchRequestUserData, - proxyConfiguration: ProxyConfiguration | undefined, + proxyOptions: ProxyOptions, startOffset = 0, ): RequestOptions { // Initialize or update pagination fields @@ -136,9 +137,7 @@ export function createSearchRequest( const currentPage = userData.currentPage ?? 0; const totalPages = userData.totalPages ?? Math.ceil(userData.maxResults / 10) + 1; - // @ts-expect-error is there a better way to get group information? - // (e.g. to create extended CheerioCrawlOptions and pass it there?) - const groups = proxyConfiguration?.groups || []; + const groups = proxyOptions.groups ?? []; const protocol = groups.includes('GOOGLE_SERP') ? 'http' : 'https'; const urlSearch = startOffset > 0 ? `${protocol}://www.google.com/search?q=${encodeURI(userData.query)}&start=${startOffset}` diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts index ec615d6..ffc091f 100644 --- a/tests/cheerio-crawler.content.test.ts +++ b/tests/cheerio-crawler.content.test.ts @@ -88,7 +88,7 @@ describe('Cheerio Crawler Content Tests', () => { it('test the crawler is created with the impit HTTP client', async () => { const { crawler } = await createAndStartContentCrawler( - { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {} }, + { type: ContentCrawlerTypes.CHEERIO, crawlerOptions: {}, proxyOptions: {} }, false, ); diff --git a/tests/crawler-key.test.ts b/tests/crawler-key.test.ts new file mode 100644 index 0000000..7a84df3 --- /dev/null +++ b/tests/crawler-key.test.ts @@ -0,0 +1,133 @@ +import type { CheerioCrawlerOptions } from 'crawlee'; +import { log } from 'crawlee'; +import { beforeAll, describe, expect, it } from 'vitest'; + +import { ContentCrawlerTypes } from '../src/const.js'; +import { getCrawlerKey } from '../src/crawlers.js'; +import { processInput, processStandbyInput } from '../src/input.js'; +import type { ProxyOptions } from '../src/types.js'; +import { parseParameters } from '../src/utils.js'; + +const baseOptions: CheerioCrawlerOptions = { + keepAlive: true, + maxRequestRetries: 1, + requestHandlerTimeoutSecs: 40, + autoscaledPoolOptions: { desiredConcurrency: 5 }, +}; + +const cheerioKey = ( + options: CheerioCrawlerOptions = {}, + proxyOptions: ProxyOptions = { useApifyProxy: true }, +) => getCrawlerKey(ContentCrawlerTypes.CHEERIO, { ...baseOptions, ...options }, proxyOptions); + +describe('getCrawlerKey', () => { + // The key doubles as a request queue name, so it has to stay a short slug. + it('is a slug of the crawler kind and a hash', () => { + expect(cheerioKey()).toMatch(/^cheerio-[0-9a-f]+$/); + }); + + it('separates the crawler kinds', () => { + const keys = new Set([ + getCrawlerKey('search', baseOptions, {}), + getCrawlerKey(ContentCrawlerTypes.CHEERIO, baseOptions, {}), + getCrawlerKey(ContentCrawlerTypes.PLAYWRIGHT, baseOptions, {}), + ]); + + expect(keys.size).toBe(3); + }); + + it('ignores the order the proxy options were declared in', () => { + const a = cheerioKey({}, { useApifyProxy: true, countryCode: 'US' }); + const b = cheerioKey({}, { countryCode: 'US', useApifyProxy: true }); + + expect(a).toBe(b); + }); + + it('treats the apifyProxy* input-schema aliases as their canonical counterparts', () => { + expect(cheerioKey({}, { apifyProxyGroups: ['RESIDENTIAL'] })).toBe(cheerioKey({}, { groups: ['RESIDENTIAL'] })); + expect(cheerioKey({}, { apifyProxyCountry: 'US' })).toBe(cheerioKey({}, { countryCode: 'US' })); + }); + + it('collapses the useApifyProxy spellings the way the SDK does', () => { + const noProxy = cheerioKey({}, { useApifyProxy: false }); + const custom = cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }); + + expect(cheerioKey({}, { useApifyProxy: true })).toBe(cheerioKey({}, {})); + expect(cheerioKey({}, { useApifyProxy: false, proxyUrls: ['http://proxy.example.com:8000'] })).toBe(custom); + expect(cheerioKey({}, { useApifyProxy: false, tieredProxyUrls: [['http://a:1']] })).toBe(noProxy); + expect(noProxy).not.toBe(custom); + }); + + it('never exposes proxy credentials, so the key is safe to log', () => { + const key = cheerioKey({}, { + password: 'hunter2', + proxyUrls: ['http://user:hunter2@proxy.example.com:8000'], + }); + + expect(key).not.toContain('hunter2'); + expect(key).not.toContain('proxy.example.com'); + }); + + it('still separates crawlers whose settings genuinely differ', () => { + const keys = new Set([ + cheerioKey(), + cheerioKey({ keepAlive: false }), + cheerioKey({ maxRequestRetries: 3 }), + cheerioKey({ requestHandlerTimeoutSecs: 90 }), + cheerioKey({ autoscaledPoolOptions: { desiredConcurrency: 10 } }), + cheerioKey({}, { groups: ['RESIDENTIAL'] }), + cheerioKey({}, { countryCode: 'US' }), + cheerioKey({}, { useApifyProxy: false }), + cheerioKey({}, { password: 'hunter2' }), + cheerioKey({}, { proxyUrls: ['http://proxy.example.com:8000'] }), + cheerioKey({}, { proxyUrls: ['http://other.example.com:8000'] }), + ]); + + expect(keys.size).toBe(11); + }); +}); + +describe('standby requests reuse the crawlers started at boot', () => { + process.env.ACTOR_FULL_NAME = 'apify/rag-web-browser'; + + // A custom proxy keeps `Actor.createProxyConfiguration` off the network and off + // `APIFY_PROXY_PASSWORD`, which it needs to return a `ProxyConfiguration` at all. + const proxyConfiguration = { useApifyProxy: false, proxyUrls: ['http://proxy.invalid:8000'] }; + const query = (extraParams = '') => `?query=hello&proxyConfiguration=${ + encodeURIComponent(JSON.stringify(proxyConfiguration))}${extraParams}`; + + let bootKeys: string[]; + + const keysForRequest = async (queryString: string) => { + const { searchCrawlerOptions, contentCrawlerOptions } = await processInput(parseParameters(queryString)); + // Mirrors `runSearchProcess`, which forces keepAlive to match the crawlers started at boot. + searchCrawlerOptions.crawlerOptions.keepAlive = true; + contentCrawlerOptions.crawlerOptions.keepAlive = true; + + return [ + getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), + getCrawlerKey(contentCrawlerOptions.type, contentCrawlerOptions.crawlerOptions, contentCrawlerOptions.proxyOptions), + ]; + }; + + beforeAll(async () => { + const { searchCrawlerOptions, contentCrawlerOptions } = await processStandbyInput({ proxyConfiguration }); + + bootKeys = [ + getCrawlerKey('search', searchCrawlerOptions.crawlerOptions, searchCrawlerOptions.proxyOptions), + ...contentCrawlerOptions.map((o) => getCrawlerKey(o.type, o.crawlerOptions, o.proxyOptions)), + ]; + expect(new Set(bootKeys).size).toBe(3); + }); + + it('reuses them for a plain request', async () => { + expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query()))); + }); + + it('reuses them when debugMode is requested, and leaves the log level alone', async () => { + const levelBefore = log.getLevel(); + + expect(bootKeys).toEqual(expect.arrayContaining(await keysForRequest(query('&debugMode=true')))); + expect(log.getLevel()).toBe(levelBefore); + }); +});