Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import type {
SERPProxyGroup,
UrlToMarkdownInput,
} from './types.js';
import { abortRun } from './utils.js';

/**
* Processes the input and returns an array of crawler settings. This is ideal for startup of STANDBY mode
Expand All @@ -27,7 +28,7 @@ import type {
export async function processStandbyInput(originalInput: Partial<Input>) {
const { input, searchCrawlerOptions, contentScraperSettings } = await processInputInternal(originalInput, true);

const proxy = await Actor.createProxyConfiguration(input.proxyConfiguration);
const proxy = await createContentProxyConfiguration(input.proxyConfiguration);
const contentCrawlerOptions: ContentCrawlerOptions[] = [
createPlaywrightCrawlerOptions(input, proxy),
createCheerioCrawlerOptions(input, proxy),
Expand All @@ -42,7 +43,7 @@ export async function processStandbyInput(originalInput: Partial<Input>) {
export async function processInput(originalInput: Partial<Input>) {
const { input, searchCrawlerOptions, contentScraperSettings } = await processInputInternal(originalInput);

const proxy = await Actor.createProxyConfiguration(input.proxyConfiguration);
const proxy = await createContentProxyConfiguration(input.proxyConfiguration);
const contentCrawlerOptions: ContentCrawlerOptions = input.scrapingTool === 'raw-http'
? createCheerioCrawlerOptions(input, proxy, false)
: createPlaywrightCrawlerOptions(input, proxy, false);
Expand Down Expand Up @@ -210,6 +211,14 @@ async function processUrlToMarkdownInput(input: Partial<UrlToMarkdownInput>): Pr
return validatedInput;
}

async function createContentProxyConfiguration(proxyConfiguration: ProxyConfigurationOptions) {
try {
return await Actor.createProxyConfiguration(proxyConfiguration);
} catch (e) {
return abortRun(`Cannot use Apify Proxy for scraping the target pages: ${(e as Error).message}`);
}
}

function createPlaywrightCrawlerOptions(
input: Input,
proxy: ProxyConfiguration | undefined,
Expand Down
16 changes: 16 additions & 0 deletions src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,22 @@ export function isActorStandby(): boolean {
return Actor.getEnv().metaOrigin === 'STANDBY';
}

/**
* Aborts the run with a terminal status message, instead of crashing or failing it.
*/
export async function abortRun(statusMessage: string): Promise<never> {
log.error(statusMessage);

if (!Actor.isAtHome()) {
process.exit(1);
}

// Aborting gracefully buys the 30 seconds that the teardown needs before the container is stopped.
await Actor.abort(Actor.getEnv().actorRunId!, { statusMessage, gracefully: true });
await Actor.exit({ exit: false });
process.exit(0);
}

/**
* Extracts the calling end-user's authorization from the x-apify-user-authorization header.
* Node lower-cases header names, and the header could theoretically arrive as a string array.
Expand Down
Loading