diff --git a/agent-quickstart/elixir.mdx b/agent-quickstart/elixir.mdx new file mode 100644 index 000000000..e95a36f1d --- /dev/null +++ b/agent-quickstart/elixir.mdx @@ -0,0 +1,211 @@ +--- +title: "Elixir Agent Quickstart" +description: "Canonical Firecrawl Elixir quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Elixir Agent Quickstart + +This file is the canonical quickstart for external agents integrating Firecrawl via the Elixir SDK. It is generated from SDK source and OpenAPI spec. + +## Install + +Add to `mix.exs`: + +```elixir +defp deps do + [ + {:firecrawl, "~> 1.9"} + ] +end +``` + +Then run: + +```bash +mix deps.get +``` + +## Authenticate + +Set the API key globally in your config: + +```elixir +# config/config.exs +config :firecrawl, api_key: "fc-YOUR_API_KEY" +``` + +Or pass it per-request: + +```elixir +Firecrawl.scrape_and_extract_from_url([url: "https://example.com"], api_key: "fc-YOUR_API_KEY") +``` + +All functions accept an optional trailing keyword list (`opts`) that supports `:api_key` and `:base_url` (for self-hosted instances, default `"https://api.firecrawl.dev/v2"`). + +## When To Use What + +- **`search_and_scrape`**: Use when you start with a query and need to discover relevant URLs and their content. +- **`scrape_and_extract_from_url`**: Use when you already have a specific URL and want its page content in markdown, HTML, JSON, or other formats. +- **`interact_with_scrape_browser_session`**: Use when the page needs post-scrape browser actions like clicking, typing, scrolling, or executing code in a live browser session. + +## Search + +### Why use it + +Search the web for a query and optionally scrape the results. Returns categorized results from web, news, and image sources with optional content extraction. + +### Preferred SDK method + +```elixir +Firecrawl.search_and_scrape(params, opts \\ []) +``` + +Bang variant: `Firecrawl.search_and_scrape!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, response} = Firecrawl.search_and_scrape( + query: "firecrawl web scraping API", + limit: 5, + highlights: true +) + +IO.inspect(response.body) +``` + +### Parameters + +All parameters are passed as a keyword list. Only `query` is required. + +| Parameter | Elixir Key | JSON Key | Type | Description | +|---|---|---|---|---| +| query | `:query` | `"query"` | `:string` | The search query (required). | +| limit | `:limit` | `"limit"` | `:integer` | Maximum number of results. Server default: `10`. | +| sources | `:sources` | `"sources"` | `{:list, :any}` | Sources to search: `"web"`, `"news"`, `"images"`. | +| categories | `:categories` | `"categories"` | `{:list, :any}` | Filter by category: `"github"`, `"research"`, `"pdf"`, `"developer"`. | +| include_domains | `:include_domains` | `"includeDomains"` | `{:list, :string}` | Restrict results to these domains. | +| exclude_domains | `:exclude_domains` | `"excludeDomains"` | `{:list, :string}` | Exclude results from these domains. | +| tbs | `:tbs` | `"tbs"` | `:string` | Time-based filter (e.g. `"qdr:d"` for past day). | +| location | `:location` | `"location"` | `:string` | Location for geo-targeted search. | +| country | `:country` | `"country"` | `:string` | ISO country code (e.g. `"US"`). | +| ignore_invalid_urls | `:ignore_invalid_urls` | `"ignoreInvalidURLs"` | `:boolean` | Ignore invalid URLs in results. | +| timeout | `:timeout` | `"timeout"` | `:integer` | Timeout in milliseconds. Server default: `60000`. | +| highlights | `:highlights` | `"highlights"` | `:boolean` | Generate query-relevant highlights. Server default: `true`. | +| scrape_options | `:scrape_options` | `"scrapeOptions"` | `:keyword_list` | Nested scrape configuration. | +| enterprise | `:enterprise` | `"enterprise"` | `{:list, :string}` | Enterprise ZDR options: `["zdr"]` or `["anon"]`. | + +## Scrape + +### Why use it + +Scrape a single URL and get its content in one or more formats. Supports browser automation, LLM extraction, screenshots, and more. + +### Preferred SDK method + +```elixir +Firecrawl.scrape_and_extract_from_url(params, opts \\ []) +``` + +Bang variant: `Firecrawl.scrape_and_extract_from_url!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, response} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com", + formats: ["markdown", "html"], + only_main_content: true +) + +IO.inspect(response.body["data"]["markdown"]) +``` + +### Parameters + +All parameters are passed as a keyword list. Only `url` is required. + +| Parameter | Elixir Key | JSON Key | Type | Description | +|---|---|---|---|---| +| url | `:url` | `"url"` | `:string` | The URL to scrape (required). | +| formats | `:formats` | `"formats"` | `{:list, :any}` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"json"`, etc. Server default: `["markdown"]`. | +| only_main_content | `:only_main_content` | `"onlyMainContent"` | `:boolean` | Strip boilerplate. Server default: `true`. | +| headers | `:headers` | `"headers"` | `:any` | Custom HTTP headers. | +| include_tags | `:include_tags` | `"includeTags"` | `{:list, :string}` | HTML tags to include. | +| exclude_tags | `:exclude_tags` | `"excludeTags"` | `{:list, :string}` | HTML tags to exclude. | +| timeout | `:timeout` | `"timeout"` | `:integer` | Timeout in ms. Min: `1000`, max: `300000`. Server default: `60000`. | +| wait_for | `:wait_for` | `"waitFor"` | `:integer` | Delay in ms before fetching content. | +| mobile | `:mobile` | `"mobile"` | `:boolean` | Emulate mobile device. | +| parsers | `:parsers` | `"parsers"` | `{:list, :any}` | Parser configurations. Server default: `["pdf"]`. | +| actions | `:actions` | `"actions"` | `{:list, :any}` | Browser automation actions. | +| location | `:location` | `"location"` | `:keyword_list` | Location settings. | +| skip_tls_verification | `:skip_tls_verification` | `"skipTlsVerification"` | `:boolean` | Skip TLS verification. | +| remove_base64_images | `:remove_base64_images` | `"removeBase64Images"` | `:boolean` | Remove base64 images. Server default: `true`. | +| block_ads | `:block_ads` | `"blockAds"` | `:boolean` | Block advertisements. Server default: `true`. | +| proxy | `:proxy` | `"proxy"` | `:basic \| :enhanced \| :auto` | Proxy type. Server default: `:auto`. | +| max_age | `:max_age` | `"maxAge"` | `:integer` | Use cached result if younger than this (ms). | +| min_age | `:min_age` | `"minAge"` | `:integer` | Cache-only mode. Set to `1` for any cached data. | +| store_in_cache | `:store_in_cache` | `"storeInCache"` | `:boolean` | Cache the result. Server default: `true`. | +| lockdown | `:lockdown` | `"lockdown"` | `:boolean` | Only serve cached results. | +| redact_pii | `:redact_pii` | `"redactPII"` | `:boolean` | Redact PII from content. | +| profile | `:profile` | `"profile"` | `:keyword_list` | Browser profile for session continuity. | +| audit_metadata | `:audit_metadata` | `"auditMetadata"` | `:keyword_list` | User attribution: `[username: "..."]`. | +| zero_data_retention | `:zero_data_retention` | `"zeroDataRetention"` | `:boolean` | Enable zero data retention. | + +## Interact + +### Why use it + +Execute code in a live browser session tied to a scrape job. Use for post-scrape interactions like clicking buttons, filling forms, or running scripts. + +### Preferred SDK method + +```elixir +Firecrawl.interact_with_scrape_browser_session(job_id, params, opts \\ []) +``` + +Bang variant: `Firecrawl.interact_with_scrape_browser_session!(job_id, params, opts)` raises on error. + +### Example + +```elixir +{:ok, scrape_response} = Firecrawl.scrape_and_extract_from_url(url: "https://example.com") +job_id = scrape_response.body["data"]["metadata"]["jobId"] + +{:ok, result} = Firecrawl.interact_with_scrape_browser_session(job_id, + code: "document.title", + language: :node +) + +IO.inspect(result.body) + +Firecrawl.stop_interactive_scrape_browser_session(job_id) +``` + +### Parameters + +The first argument is the `job_id` (string). Remaining parameters are passed as a keyword list. + +| Parameter | Elixir Key | JSON Key | Type | Description | +|---|---|---|---|---| +| job_id | first argument | URL path | `String.t()` | The scrape job ID (required). | +| code | `:code` | `"code"` | `:string` | Code to execute in the browser sandbox (required). | +| language | `:language` | `"language"` | `:python \| :node \| :bash` | Execution language. Server default: `:node`. | +| timeout | `:timeout` | `"timeout"` | `:integer` | Execution timeout in seconds. Range: 1-300. Server default: `30`. | +| origin | `:origin` | `"origin"` | `:string` | Origin label for telemetry. | + +Use `Firecrawl.stop_interactive_scrape_browser_session(job_id)` to end the browser session. + +## Notes + +- **Naming style**: Parameters use snake_case atom keys. The SDK converts them to camelCase JSON keys automatically. +- **OpenAPI-generated**: The Elixir SDK is auto-generated from the OpenAPI spec. Function names match the OpenAPI operation IDs directly (e.g. `scrape_and_extract_from_url`, not `scrape`). +- **Return type**: All functions return `{:ok, %Req.Response{}}` or `{:error, exception}`. The bang variants (`!`) raise on error and return `Req.Response.t()` directly. HTTP 4xx/5xx responses raise `Firecrawl.Error` with `:status` and `:body` fields. +- **No `prompt` support**: Unlike JS/Python/Rust, the Elixir SDK `interact_with_scrape_browser_session` only accepts `code`, not a natural-language `prompt`. +- **Origin auto-injection**: The SDK automatically injects `"origin": "elixir-sdk@{version}"` into every request body. + +## Source Of Truth + +- `/firecrawl/apps/elixir-sdk/lib/firecrawl.ex` +- `/firecrawl/apps/elixir-sdk/mix.exs` +- `/firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/java.mdx b/agent-quickstart/java.mdx new file mode 100644 index 000000000..9a611c1bb --- /dev/null +++ b/agent-quickstart/java.mdx @@ -0,0 +1,235 @@ +--- +title: "Java Agent Quickstart" +description: "Canonical Firecrawl Java quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Java Agent Quickstart + +This file is the canonical quickstart for external agents integrating Firecrawl via the Java SDK. It is generated from SDK source and OpenAPI spec. + +## Install + +**Gradle:** + +```groovy +implementation("com.firecrawl:firecrawl-java:1.12.1") +``` + +**Maven:** + +```xml + + com.firecrawl + firecrawl-java + 1.12.1 + +``` + +Requires Java 11+. + +## Authenticate + +```java +import com.firecrawl.client.FirecrawlClient; + +FirecrawlClient client = FirecrawlClient.builder() + .apiKey("fc-YOUR_API_KEY") + .build(); +``` + +Or from environment variable (`FIRECRAWL_API_KEY`): + +```java +FirecrawlClient client = FirecrawlClient.fromEnv(); +``` + +Builder options: `apiKey` (String), `apiUrl` (String, default `"https://api.firecrawl.dev"`), `timeoutMs` (long, default `300000`), `maxRetries` (int, default `3`), `backoffFactor` (double, default `0.5`), `asyncExecutor` (Executor), `httpClient` (OkHttpClient). + +## When To Use What + +- **`search`**: Use when you start with a query and need to discover relevant URLs and their content. Returns results from web, news, and image sources. +- **`scrape`**: Use when you already have a specific URL and want its page content in markdown, HTML, JSON, or other formats. +- **`interact`**: Use when the page needs post-scrape browser actions like clicking, typing, scrolling, or executing code in a live browser session. + +## Search + +### Why use it + +Search the web for a query and optionally scrape the results. Returns categorized results from web, news, and image sources with optional content extraction. + +### Preferred SDK method + +```java +client.search(query) +client.search(query, options) +``` + +### Example + +```java +import com.firecrawl.models.SearchOptions; +import com.firecrawl.models.SearchData; + +SearchData results = client.search("firecrawl web scraping API", + SearchOptions.builder() + .limit(5) + .highlights(true) + .build()); + +for (var item : results.getWeb()) { + System.out.println(item.get("title") + " " + item.get("url")); +} +``` + +### Parameters + +All `SearchOptions` fields are `null` by default (omitted from the request body). Use the builder pattern. + +| Parameter | Type | Description | +|---|---|---| +| `query` | `String` | The search query (first argument, required). | +| `sources` | `List` | Sources to search: `"web"`, `"news"`, `"images"`. | +| `categories` | `List` | Filter by category: `"github"`, `"research"`, `"pdf"`. | +| `includeDomains` | `List` | Restrict results to these domains. | +| `excludeDomains` | `List` | Exclude results from these domains. | +| `limit` | `Integer` | Maximum number of results. Server default: `10`. | +| `tbs` | `String` | Time-based search filter (e.g. `"qdr:d"` for past day). | +| `location` | `String` | Location string for geo-targeted search. | +| `ignoreInvalidURLs` | `Boolean` | Ignore invalid URLs in results. | +| `timeout` | `Integer` | Timeout in milliseconds. Server default: `60000`. | +| `highlights` | `Boolean` | Generate query-relevant highlights. Server default: `true`. | +| `scrapeOptions` | `ScrapeOptions` | Nested scrape configuration applied to each result. | +| `integration` | `String` | Integration identifier. | + +### Return type + +`SearchData` with: `getWeb()`, `getNews()`, `getImages()`. Each returns `List>`. + +## Scrape + +### Why use it + +Scrape a single URL and get its content in one or more formats. Supports browser automation, LLM extraction, screenshots, and more. + +### Preferred SDK method + +```java +client.scrape(url) +client.scrape(url, options) +``` + +### Example + +```java +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.Document; + +Document doc = client.scrape("https://example.com", + ScrapeOptions.builder() + .formats(List.of("markdown", "html")) + .onlyMainContent(true) + .build()); + +System.out.println(doc.getMarkdown()); +``` + +### Parameters + +All `ScrapeOptions` fields are `null` by default (omitted from the request body). Use the builder pattern. + +| Parameter | Type | Description | +|---|---|---| +| `url` | `String` | The URL to scrape (first argument, required). | +| `formats` | `List` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"json"`, `"audio"`, `"video"`, or typed objects (`JsonFormat`, `QuestionFormat`, `HighlightsFormat`). Server default: `["markdown"]`. | +| `onlyMainContent` | `Boolean` | Strip boilerplate, keep main content only. Server default: `true`. | +| `headers` | `Map` | Custom HTTP headers. | +| `includeTags` | `List` | HTML tags to exclusively include. | +| `excludeTags` | `List` | HTML tags to exclude. | +| `timeout` | `Integer` | Timeout in milliseconds. Server default: `60000`. | +| `waitFor` | `Integer` | Delay in milliseconds before fetching content. | +| `mobile` | `Boolean` | Emulate mobile device. | +| `parsers` | `List` | Parser configurations (e.g. `"pdf"` or `Map.of("type", "pdf", "maxPages", 10)`). | +| `actions` | `List>` | Browser automation actions as maps. | +| `location` | `LocationConfig` | Location settings: `LocationConfig.builder().country("US").languages(List.of("en-US")).build()`. | +| `skipTlsVerification` | `Boolean` | Skip TLS certificate verification. | +| `removeBase64Images` | `Boolean` | Remove base64 images from markdown. Server default: `true`. | +| `blockAds` | `Boolean` | Block advertisements. Server default: `true`. | +| `proxy` | `String` | Proxy mode: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. Server default: `"auto"`. | +| `maxAge` | `Long` | Use cached result if younger than this (ms). | +| `storeInCache` | `Boolean` | Cache the result. Server default: `true`. | +| `lockdown` | `Boolean` | Only serve cached results. | +| `redactPII` | `Boolean` | Redact PII from returned content. | +| `auditMetadata` | `AuditMetadata` | User attribution: `new AuditMetadata("username")`. | +| `integration` | `String` | Integration identifier. | + +#### Format objects + +| Type | Constructor | Description | +|---|---|---| +| `JsonFormat` | `JsonFormat.builder().prompt("...").schema(Map.of(...)).build()` | LLM-extracted structured JSON. | +| `QuestionFormat` | `new QuestionFormat("your question")` | Ask a question about the page. | +| `HighlightsFormat` | `new HighlightsFormat("your query")` | Find relevant source text. | + +## Interact + +### Why use it + +Execute code in a live browser session tied to a scrape job. Use for post-scrape interactions like clicking buttons, filling forms, or running scripts. + +### Preferred SDK method + +```java +client.interact(jobId, code) +client.interact(jobId, code, language, timeout) +client.interact(jobId, code, language, timeout, origin) +``` + +### Example + +```java +import com.firecrawl.models.BrowserExecuteResponse; + +Document doc = client.scrape("https://example.com"); +String jobId = (String) doc.getMetadata().get("jobId"); + +BrowserExecuteResponse result = client.interact(jobId, "document.title"); +System.out.println(result.getStdout()); + +client.stopInteractiveBrowser(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `String` | The scrape job ID (required). | +| `code` | `String` | Code to execute in the browser sandbox (required). | +| `language` | `String` | Execution language: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `timeout` | `Integer` | Execution timeout in seconds. Range: 1-300. Server default: `30`. | +| `origin` | `String` | Origin identifier. Auto-set to `"java-sdk@{version}"` if absent. | + +Use `client.stopInteractiveBrowser(jobId)` to end the browser session. + +### Async variants + +All three methods have `Async` variants returning `CompletableFuture`: + +```java +client.scrapeAsync(url, options) +client.searchAsync(query, options) +client.interactAsync(jobId, code, language, timeout, origin) +``` + +## Notes + +- **Naming style**: All parameters use camelCase. Options use the builder pattern. +- **Deprecated aliases**: `scrapeExecute()` is deprecated in favor of `interact()`. `deleteScrapeBrowser()` is deprecated in favor of `stopInteractiveBrowser()`. +- **No `prompt` support**: Unlike JS/Python/Rust, the Java SDK `interact()` method only accepts `code`, not natural-language `prompt`. Use the `code` parameter with `"bash"` language for agent-browser CLI commands. +- **Null handling**: All option fields use `@JsonInclude(NON_NULL)`, so `null` fields are omitted from the request body. + +## Source Of Truth + +- `/firecrawl/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java` +- `/firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java` +- `/firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java` +- `/firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/node.mdx b/agent-quickstart/node.mdx new file mode 100644 index 000000000..7d0d6398f --- /dev/null +++ b/agent-quickstart/node.mdx @@ -0,0 +1,216 @@ +--- +title: "Node.js Agent Quickstart" +description: "Canonical Firecrawl Node.js quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Node.js Agent Quickstart + +This file is the canonical quickstart for external agents integrating Firecrawl via the Node.js/TypeScript SDK. It is generated from SDK source and OpenAPI spec. + +## Install + +```bash +npm install @mendable/firecrawl-js +``` + +## Authenticate + +```typescript +import Firecrawl from "@mendable/firecrawl-js"; + +const client = new Firecrawl({ apiKey: "fc-YOUR_API_KEY" }); +``` + +The API key can also be set via the `FIRECRAWL_API_KEY` environment variable. If omitted, the client falls back to keyless free tier (rate-limited per IP). + +## When To Use What + +- **`search`**: Use when you start with a query and need to discover relevant URLs and their content. Returns results from web, news, and image sources. +- **`scrape`**: Use when you already have a specific URL and want its page content in markdown, HTML, JSON, or other formats. +- **`interact`**: Use when the page needs post-scrape browser actions like clicking, typing, scrolling, or executing code in a live browser session. + +## Search + +### Why use it + +Search the web for a query and optionally scrape the results. Returns categorized results from web, news, and image sources with optional content extraction. + +### Preferred SDK method + +```typescript +client.search(query, options?) +``` + +### Example + +```typescript +const results = await client.search("firecrawl web scraping API", { + limit: 5, + scrapeOptions: { + formats: ["markdown"], + }, +}); + +for (const item of results.web ?? []) { + console.log(item.title, item.url); +} +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `string` | The search query (first argument, required). | +| `sources` | `Array<"web" \| "news" \| "images">` | Which search indices to query. | +| `categories` | `Array<"github" \| "research" \| "pdf" \| "developer">` | Filter results by category. | +| `includeDomains` | `string[]` | Restrict results to these domains. Cannot be used with `excludeDomains`. | +| `excludeDomains` | `string[]` | Exclude results from these domains. Cannot be used with `includeDomains`. | +| `limit` | `number` | Maximum number of results to return. Server default: `10`. | +| `tbs` | `string` | Time-based search filter (e.g. `"qdr:d"` for past day, `"qdr:w"` for past week). | +| `location` | `string` | Location string for geo-targeted search. | +| `ignoreInvalidURLs` | `boolean` | Ignore invalid URLs in results. | +| `timeout` | `number` | Timeout in milliseconds. Server default: `60000`. | +| `highlights` | `boolean` | Generate query-relevant highlights. Default: `true`. | +| `scrapeOptions` | `ScrapeOptions` | Nested scrape configuration applied to each result. | +| `enterprise` | `Array<"default" \| "anon" \| "zdr">` | Enterprise search options for Zero Data Retention. | +| `threatProtection` | `ThreatProtectionOptions` | Per-request threat protection overrides. | +| `integration` | `string` | Integration identifier. | +| `origin` | `string` | Origin identifier. | + +## Scrape + +### Why use it + +Scrape a single URL and get its content in one or more formats. Supports browser automation, LLM extraction, screenshots, and more. + +### Preferred SDK method + +```typescript +client.scrape(url, options?) +``` + +### Example + +```typescript +const doc = await client.scrape("https://example.com", { + formats: ["markdown", "html"], + onlyMainContent: true, +}); + +console.log(doc.markdown); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `string` | The URL to scrape (first argument, required). | +| `formats` | `FormatOption[]` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"product"`, `"menu"`, `"audio"`, `"video"`, or typed objects (see below). Server default: `["markdown"]`. | +| `onlyMainContent` | `boolean` | Strip boilerplate, keep main content only. Server default: `true`. | +| `headers` | `Record` | Custom HTTP headers to send with the request. | +| `includeTags` | `string[]` | HTML tags to exclusively include. | +| `excludeTags` | `string[]` | HTML tags to exclude. | +| `timeout` | `number` | Timeout in milliseconds. Min: `1000`, max: `300000`. Server default: `60000`. | +| `waitFor` | `number` | Delay in milliseconds before fetching content. Server default: `0`. | +| `mobile` | `boolean` | Emulate mobile device viewport and user-agent. Server default: `false`. | +| `parsers` | `Array` | Parser configurations (e.g. `"pdf"` or `{ type: "pdf", mode: "auto", maxPages: 10 }`). Server default: `["pdf"]`. | +| `actions` | `ActionOption[]` | Browser automation actions to perform before scraping. See Actions below. | +| `location` | `LocationConfig` | Location settings: `{ country?: string, languages?: string[] }`. Default country: `"US"`. | +| `skipTlsVerification` | `boolean` | Skip TLS certificate verification. | +| `removeBase64Images` | `boolean` | Remove base64-encoded images from markdown output. Server default: `true`. | +| `fastMode` | `boolean` | Enable fast mode for faster scrapes with reduced accuracy. | +| `blockAds` | `boolean` | Block advertisements and cookie popups. Server default: `true`. | +| `proxy` | `"basic" \| "stealth" \| "enhanced" \| "auto"` | Proxy tier. `"basic"`: fast. `"enhanced"`: advanced anti-bot (up to 5 credits). `"auto"`: tries basic first, retries with enhanced. Server default: `"auto"`. | +| `maxAge` | `number` | Use cached result if younger than this many milliseconds. Server default: `172800000` (2 days). | +| `minAge` | `number` | Cache-only mode. Set to `1` for any cached data. Returns 404 on cache miss. | +| `storeInCache` | `boolean` | Cache the scrape result. Server default: `true`. | +| `lockdown` | `boolean` | Only serve cached results, never make outbound requests. | +| `redactPII` | `boolean \| RedactPIIOptions` | Redact personally identifiable information. Pass `true` for defaults or an options object. | +| `threatProtection` | `ThreatProtectionOptions` | Per-request threat protection overrides. | +| `auditMetadata` | `{ username: string }` | User attribution for SIEM logging. | +| `profile` | `{ name: string, saveChanges?: boolean }` | Persistent browser profile for session continuity. | +| `integration` | `string` | Integration identifier. | +| `origin` | `string` | Origin identifier. | +| `useMock` | `string` | Use mock data. | + +#### Format objects + +| Type | Fields | +|---|---| +| `JsonFormat` | `{ type: "json", prompt?: string, schema?: object \| ZodSchema }` | +| `ScreenshotFormat` | `{ type: "screenshot", fullPage?: boolean, quality?: number, viewport?: { width, height } }` | +| `ChangeTrackingFormat` | `{ type: "changeTracking", modes: ("git-diff" \| "json")[], schema?: object, prompt?: string, tag?: string }` | +| `AttributesFormat` | `{ type: "attributes", selectors: { selector: string, attribute: string }[] }` | +| `QuestionFormat` | `{ type: "question", question: string }` | +| `HighlightsFormat` | `{ type: "highlights", query: string }` | + +#### Actions + +| Type | Fields | +|---|---| +| `wait` | `{ type: "wait", milliseconds?: number, selector?: string }` | +| `screenshot` | `{ type: "screenshot", fullPage?: boolean, quality?: number, viewport?: { width, height } }` | +| `click` | `{ type: "click", selector: string }` | +| `write` | `{ type: "write", text: string }` | +| `press` | `{ type: "press", key: string }` | +| `scroll` | `{ type: "scroll", direction: "up" \| "down", selector?: string }` | +| `scrape` | `{ type: "scrape" }` | +| `executeJavascript` | `{ type: "executeJavascript", script: string }` | +| `pdf` | `{ type: "pdf", format?: string, landscape?: boolean, scale?: number }` | + +## Interact + +### Why use it + +Execute code or send natural-language prompts in a live browser session tied to a scrape job. Use for post-scrape interactions like clicking buttons, filling forms, or running scripts. + +### Preferred SDK method + +```typescript +client.interact(jobId, args) +``` + +### Example + +```typescript +const doc = await client.scrape("https://example.com", { + formats: ["markdown"], +}); + +const jobId = doc.metadata?.jobId; + +const result = await client.interact(jobId, { + code: "document.title", + language: "node", +}); + +console.log(result.stdout); + +await client.stopInteraction(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `string` | The scrape job ID (first argument, required). | +| `code` | `string` | Code to execute in the browser sandbox. | +| `prompt` | `string` | Natural-language instruction for the browser agent. | +| `language` | `"python" \| "node" \| "bash"` | Execution language. Server default: `"node"`. | +| `timeout` | `number` | Execution timeout in seconds. Range: 1-300. Server default: `30`. | +| `origin` | `string` | Origin identifier for telemetry. | + +Use `client.stopInteraction(jobId)` to end the browser session when done. + +## Notes + +- **Naming style**: All parameters use camelCase. +- **Deprecated aliases**: `scrapeUrl()` is deprecated in favor of `scrape()`. `scrapeExecute()` is deprecated in favor of `interact()`. `stopInteractiveBrowser()` and `deleteScrapeBrowser()` are deprecated in favor of `stopInteraction()`. +- **Zod schema support**: The `scrape()` method accepts Zod schemas in `JsonFormat` and will narrow the return type accordingly. +- **Async**: All methods return Promises. + +## Source Of Truth + +- `/firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` +- `/firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` +- `/firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/python.mdx b/agent-quickstart/python.mdx new file mode 100644 index 000000000..65eb47fdb --- /dev/null +++ b/agent-quickstart/python.mdx @@ -0,0 +1,202 @@ +--- +title: "Python Agent Quickstart" +description: "Canonical Firecrawl Python quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Python Agent Quickstart + +This file is the canonical quickstart for external agents integrating Firecrawl via the Python SDK. It is generated from SDK source and OpenAPI spec. + +## Install + +```bash +pip install firecrawl-py +``` + +## Authenticate + +```python +from firecrawl import Firecrawl + +client = Firecrawl(api_key="fc-YOUR_API_KEY") +``` + +The API key can also be set via the `FIRECRAWL_API_KEY` environment variable. If omitted, the client falls back to keyless free tier (rate-limited per IP). + +Constructor options: `api_key` (str), `api_url` (str, default `"https://api.firecrawl.dev"`), `timeout` (float), `max_retries` (int, default `3`), `backoff_factor` (float, default `0.5`). + +An async client is also available: `from firecrawl import AsyncFirecrawl`. + +## When To Use What + +- **`search`**: Use when you start with a query and need to discover relevant URLs and their content. Returns results from web, news, and image sources. +- **`scrape`**: Use when you already have a specific URL and want its page content in markdown, HTML, JSON, or other formats. +- **`interact`**: Use when the page needs post-scrape browser actions like clicking, typing, scrolling, or executing code in a live browser session. + +## Search + +### Why use it + +Search the web for a query and optionally scrape the results. Returns categorized results from web, news, and image sources with optional content extraction. + +### Preferred SDK method + +```python +client.search(query, **kwargs) +``` + +### Example + +```python +results = client.search( + "firecrawl web scraping API", + limit=5, + scrape_options={"formats": ["markdown"]}, +) + +for item in results.web or []: + print(item.get("title"), item.get("url")) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `str` | The search query (first argument, required). | +| `sources` | `list[str]` | Sources to search: `"web"`, `"news"`, `"images"`. | +| `categories` | `list[str]` | Filter by category: `"github"`, `"research"`, `"pdf"`, `"developer"`. | +| `include_domains` | `list[str]` | Restrict results to these domains. Cannot be used with `exclude_domains`. | +| `exclude_domains` | `list[str]` | Exclude results from these domains. Cannot be used with `include_domains`. | +| `limit` | `int` | Maximum number of results. Server default: `10`. | +| `tbs` | `str` | Time-based search filter (e.g. `"qdr:d"` for past day, `"qdr:w"` for past week). | +| `location` | `str` | Location string for geo-targeted search. | +| `ignore_invalid_urls` | `bool` | Ignore invalid URLs in results. | +| `timeout` | `int` | Timeout in milliseconds. Server default: `60000`. | +| `highlights` | `bool` | Generate query-relevant highlights. Default: `True`. | +| `scrape_options` | `ScrapeOptions` | Nested scrape configuration applied to each result. | +| `enterprise` | `list[str]` | Enterprise search options for ZDR. | +| `threat_protection` | `ThreatProtectionOptions` | Per-request threat protection overrides. | +| `integration` | `str` | Integration identifier. | + +### Return type + +`SearchData` with attributes: `web`, `news`, `images`, `developer`. Each is a list of result dicts or `None`. Access results via `results.web`, not `results.data`. + +## Scrape + +### Why use it + +Scrape a single URL and get its content in one or more formats. Supports browser automation, LLM extraction, screenshots, and more. + +### Preferred SDK method + +```python +client.scrape(url, **kwargs) +``` + +### Example + +```python +doc = client.scrape( + "https://example.com", + formats=["markdown", "html"], + only_main_content=True, +) + +print(doc.markdown) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `str` | The URL to scrape (first argument, required). | +| `formats` | `list` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"product"`, `"menu"`, `"audio"`, `"video"`, or typed dicts. Server default: `["markdown"]`. | +| `only_main_content` | `bool` | Strip boilerplate, keep main content only. Server default: `True`. | +| `headers` | `dict[str, str]` | Custom HTTP headers to send with the request. | +| `include_tags` | `list[str]` | HTML tags to exclusively include. | +| `exclude_tags` | `list[str]` | HTML tags to exclude. | +| `timeout` | `int` | Timeout in milliseconds. Min: `1000`, max: `300000`. Server default: `60000`. | +| `wait_for` | `int` | Delay in milliseconds before fetching content. | +| `mobile` | `bool` | Emulate mobile device. | +| `parsers` | `list` | Parser configurations (e.g. `"pdf"` or `{"type": "pdf", "mode": "auto"}`). Server default: `["pdf"]`. | +| `actions` | `list[dict]` | Browser automation actions. See Actions table below. | +| `location` | `Location` | Location settings: `{"country": "US", "languages": ["en-US"]}`. | +| `skip_tls_verification` | `bool` | Skip TLS certificate verification. | +| `remove_base64_images` | `bool` | Remove base64 images from markdown output. Server default: `True`. | +| `fast_mode` | `bool` | Enable fast mode for faster scrapes. | +| `block_ads` | `bool` | Block advertisements and cookie popups. Server default: `True`. | +| `proxy` | `str` | Proxy tier: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. Server default: `"auto"`. | +| `max_age` | `int` | Use cached result if younger than this many milliseconds. Server default: `172800000` (2 days). | +| `store_in_cache` | `bool` | Cache the scrape result. Server default: `True`. | +| `lockdown` | `bool` | Only serve cached results, never make outbound requests. | +| `threat_protection` | `ThreatProtectionOptions` | Per-request threat protection overrides. | +| `audit_metadata` | `AuditMetadata` | User attribution for SIEM logging: `{"username": "..."}`. | +| `profile` | `dict` | Persistent browser profile: `{"name": "...", "save_changes": True}`. | +| `integration` | `str` | Integration identifier. | +| `use_mock` | `str` | Use mock data. | + +#### Actions + +| Type | Fields | +|---|---| +| `wait` | `{"type": "wait", "milliseconds": int}` or `{"type": "wait", "selector": str}` | +| `screenshot` | `{"type": "screenshot", "fullPage": bool, "quality": int}` | +| `click` | `{"type": "click", "selector": str}` | +| `write` | `{"type": "write", "text": str}` | +| `press` | `{"type": "press", "key": str}` | +| `scroll` | `{"type": "scroll", "direction": "up" \| "down", "selector": str}` | +| `scrape` | `{"type": "scrape"}` | +| `executeJavascript` | `{"type": "executeJavascript", "script": str}` | +| `pdf` | `{"type": "pdf", "format": str, "landscape": bool, "scale": float}` | + +## Interact + +### Why use it + +Execute code or send natural-language prompts in a live browser session tied to a scrape job. Use for post-scrape interactions like clicking buttons, filling forms, or running scripts. + +### Preferred SDK method + +```python +client.interact(job_id, code=None, *, prompt=None, language="node", timeout=None, origin=None) +``` + +### Example + +```python +doc = client.scrape("https://example.com", formats=["markdown"]) +job_id = doc.metadata.get("jobId") + +result = client.interact(job_id, code="document.title", language="node") +print(result.stdout) + +client.stop_interaction(job_id) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `str` | The scrape job ID (first argument, required). | +| `code` | `str` | Code to execute in the browser sandbox (positional). | +| `prompt` | `str` | Natural-language instruction for the browser agent (keyword-only). | +| `language` | `str` | Execution language: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `timeout` | `int` | Execution timeout in seconds. Range: 1-300. Server default: `30`. | +| `origin` | `str` | Origin identifier for telemetry. | + +Either `code` or `prompt` must be provided. Use `client.stop_interaction(job_id)` to end the browser session. + +## Notes + +- **Naming style**: All parameters use snake_case. +- **Deprecated aliases**: `scrape_url()` is deprecated in favor of `scrape()`. `scrape_execute()` is deprecated in favor of `interact()`. `stop_interactive_browser()` and `delete_scrape_browser()` are deprecated in favor of `stop_interaction()`. `FirecrawlApp` is deprecated in favor of `Firecrawl`. +- **Async support**: Use `AsyncFirecrawl` for async/await usage with the same method signatures. +- **Search return type**: Access results via `results.web`, `results.news`, `results.images`, `results.developer`. Accessing `results.data` raises `AttributeError` with guidance. + +## Source Of Truth + +- `/firecrawl/apps/python-sdk/firecrawl/client.py` +- `/firecrawl/apps/python-sdk/firecrawl/v2/client.py` +- `/firecrawl/apps/python-sdk/firecrawl/v2/types.py` +- `/firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/rust.mdx b/agent-quickstart/rust.mdx new file mode 100644 index 000000000..1cc4a3a45 --- /dev/null +++ b/agent-quickstart/rust.mdx @@ -0,0 +1,253 @@ +--- +title: "Rust Agent Quickstart" +description: "Canonical Firecrawl Rust quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Rust Agent Quickstart + +This file is the canonical quickstart for external agents integrating Firecrawl via the Rust SDK. It is generated from SDK source and OpenAPI spec. + +## Install + +Add to `Cargo.toml`: + +```toml +[dependencies] +firecrawl = "2" +tokio = { version = "1", features = ["full"] } +``` + +## Authenticate + +```rust +use firecrawl::Client; + +let client = Client::new("fc-YOUR_API_KEY")?; +``` + +For self-hosted instances: + +```rust +let client = Client::new_selfhosted("https://your-instance.com", Some("fc-YOUR_API_KEY"))?; +``` + +The API key is optional for `new_selfhosted`. If omitted, the client falls back to keyless free tier (rate-limited per IP). + +## When To Use What + +- **`search`**: Use when you start with a query and need to discover relevant URLs and their content. Returns results from web, news, and image sources. +- **`scrape`**: Use when you already have a specific URL and want its page content in markdown, HTML, JSON, or other formats. +- **`interact`**: Use when the page needs post-scrape browser actions like clicking, typing, scrolling, or executing code in a live browser session. + +## Search + +### Why use it + +Search the web for a query and optionally scrape the results. Returns categorized results from web, news, and image sources with optional content extraction. + +### Preferred SDK method + +```rust +client.search(query, options).await +``` + +### Example + +```rust +use firecrawl::{Client, SearchOptions}; + +let client = Client::new("fc-YOUR_API_KEY")?; + +let response = client.search("firecrawl web scraping API", SearchOptions { + limit: Some(5), + ..Default::default() +}).await?; + +if let Some(web_results) = response.data.web { + for result in web_results { + println!("{:?}", result); + } +} +``` + +### Parameters + +All fields on `SearchOptions` are `Option` and default to `None` via `#[derive(Default)]`. + +| Parameter | Type | Description | +|---|---|---| +| `query` | `impl AsRef` | The search query (first argument, required). | +| `limit` | `Option` | Maximum number of results. Server default: `10`. Max: `100`. | +| `sources` | `Option>` | Sources to search: `Web`, `News`, `Images`. | +| `categories` | `Option>` | Filter by category: `Github`, `Research`, `Pdf`. | +| `include_domains` | `Option>` | Restrict results to these domains. | +| `exclude_domains` | `Option>` | Exclude results from these domains. | +| `tbs` | `Option` | Time-based search filter (e.g. `"qdr:d"` for past day). | +| `location` | `Option` | Location string for geo-targeted search. | +| `ignore_invalid_urls` | `Option` | Ignore invalid URLs in results. | +| `timeout` | `Option` | Timeout in milliseconds. Server default: `60000`. | +| `highlights` | `Option` | Generate query-relevant highlights. Server default: `true`. | +| `scrape_options` | `Option` | Nested scrape configuration applied to each result. | +| `integration` | `Option` | Integration identifier. | +| `origin` | `Option` | Auto-set to `"rust-sdk@{version}"` if `None`. | + +### Convenience method + +```rust +client.search_and_scrape(query, limit).await +``` + +Sets `scrape_options` to defaults and returns `Vec` directly, filtering out non-document results. + +## Scrape + +### Why use it + +Scrape a single URL and get its content in one or more formats. Supports browser automation, LLM extraction, screenshots, and more. + +### Preferred SDK method + +```rust +client.scrape(url, options).await +``` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format}; + +let client = Client::new("fc-YOUR_API_KEY")?; + +let doc = client.scrape("https://example.com", ScrapeOptions { + formats: Some(vec![Format::Markdown, Format::Html]), + only_main_content: Some(true), + ..Default::default() +}).await?; + +println!("{}", doc.markdown.unwrap_or_default()); +``` + +### Parameters + +All fields on `ScrapeOptions` are `Option` and default to `None` via `#[derive(Default)]`. + +| Parameter | Type | Description | +|---|---|---| +| `url` | `impl AsRef` | The URL to scrape (first argument, required). | +| `formats` | `Option>` | Output formats: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Product`, `Menu`, `Audio`, `Video`, `Question(QuestionFormat)`, `Highlights(HighlightsFormat)`. Server default: `[Markdown]`. | +| `only_main_content` | `Option` | Strip boilerplate, keep main content only. Server default: `true`. | +| `headers` | `Option>` | Custom HTTP headers. | +| `include_tags` | `Option>` | HTML tags to exclusively include. | +| `exclude_tags` | `Option>` | HTML tags to exclude. | +| `timeout` | `Option` | Timeout in milliseconds. Server default: `60000`. | +| `wait_for` | `Option` | Delay in milliseconds before fetching content. | +| `mobile` | `Option` | Emulate mobile device. | +| `parsers` | `Option>` | Parser configurations (e.g. PDF parser with mode/maxPages). | +| `actions` | `Option>` | Browser automation actions. See Actions below. | +| `location` | `Option` | Location settings: `{ country, languages }`. | +| `skip_tls_verification` | `Option` | Skip TLS certificate verification. | +| `remove_base64_images` | `Option` | Remove base64 images from markdown. Server default: `true`. | +| `fast_mode` | `Option` | Enable fast mode. | +| `block_ads` | `Option` | Block advertisements. Server default: `true`. | +| `proxy` | `Option` | Proxy type: `Basic`, `Stealth`, `Enhanced`, `Auto`. Server default: `Auto`. | +| `max_age` | `Option` | Use cached result if younger than this (ms). | +| `min_age` | `Option` | Cache-only mode; value is min age in ms. | +| `store_in_cache` | `Option` | Cache the result. Server default: `true`. | +| `lockdown` | `Option` | Only serve cached results. | +| `redact_pii` | `Option` | Redact PII from returned content. | +| `audit_metadata` | `Option` | User attribution: `AuditMetadata { username }`. | +| `profile` | `Option` | Browser profile: `ProfileConfig { name, save_changes }`. | +| `integration` | `Option` | Integration identifier. | +| `json_options` | `Option` | JSON extraction options: `{ schema, system_prompt, prompt }`. | +| `screenshot_options` | `Option` | Screenshot options: `{ full_page, quality, viewport }`. | +| `change_tracking_options` | `Option` | Change tracking: `{ modes, schema, prompt, tag }`. | +| `attribute_selectors` | `Option>` | Attribute extraction: `{ selector, attribute }`. | +| `origin` | `Option` | Auto-set to `"rust-sdk@{version}"` if `None`. | + +#### Actions (enum variants) + +| Variant | Fields | +|---|---| +| `Action::Wait` | `{ milliseconds: Option, selector: Option }` | +| `Action::Screenshot` | `{ full_page: Option, quality: Option, viewport: Option }` | +| `Action::Click` | `{ selector: String }` | +| `Action::Write` | `{ text: String }` | +| `Action::Press` | `{ key: String }` | +| `Action::Scroll` | `{ direction: ScrollDirection, selector: Option }` | +| `Action::Scrape` | (no fields) | +| `Action::ExecuteJavascript` | `{ script: String }` | +| `Action::Pdf` | `{ format: Option, landscape: Option, scale: Option }` | + +### Convenience method + +```rust +client.scrape_with_schema(url, schema, prompt).await +``` + +Extracts structured JSON using a JSON Schema and optional prompt. Returns `serde_json::Value`. + +## Interact + +### Why use it + +Execute code or send natural-language prompts in a live browser session tied to a scrape job. Use for post-scrape interactions like clicking buttons, filling forms, or running scripts. + +### Preferred SDK method + +```rust +client.interact(job_id, options).await +``` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, ScrapeExecuteOptions}; + +let client = Client::new("fc-YOUR_API_KEY")?; + +let doc = client.scrape("https://example.com", ScrapeOptions { + ..Default::default() +}).await?; + +let job_id = doc.metadata.get("jobId").unwrap().as_str().unwrap(); + +let result = client.interact(job_id, ScrapeExecuteOptions { + code: Some("document.title".to_string()), + ..Default::default() +}).await?; + +println!("{:?}", result.stdout); + +client.stop_interaction(job_id).await?; +``` + +### Parameters + +All fields on `ScrapeExecuteOptions` are `Option` and default to `None`. + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `impl AsRef` | The scrape job ID (first argument, required). | +| `code` | `Option` | Code to execute in the browser sandbox. | +| `prompt` | `Option` | Natural-language instruction for the browser agent. | +| `language` | `Option` | Execution language: `Python`, `Node`, `Bash`. Default: `Node`. | +| `timeout` | `Option` | Execution timeout in seconds. Range: 1-300. Server default: `30`. | +| `origin` | `Option` | Auto-set to `"rust-sdk@{version}"` if `None`. | + +At least one of `code` or `prompt` must be a non-empty string, or a `FirecrawlError::Misuse` is returned before any HTTP call. + +Use `client.stop_interaction(job_id).await?` to end the browser session. + +## Notes + +- **Naming style**: All struct fields use snake_case. Serde handles the camelCase conversion for the API. +- **Deprecated aliases**: `scrape_execute()` is deprecated in favor of `interact()`. `stop_interactive_browser()` and `delete_scrape_browser()` are deprecated in favor of `stop_interaction()`. +- **Async**: All methods are async and require a tokio runtime. +- **Error handling**: All methods return `Result`. + +## Source Of Truth + +- `/firecrawl/apps/rust-sdk/src/v2/client.rs` +- `/firecrawl/apps/rust-sdk/src/v2/search.rs` +- `/firecrawl/apps/rust-sdk/src/v2/scrape.rs` +- `/firecrawl-docs/api-reference/v2-openapi.json` diff --git a/docs.json b/docs.json index 5b1ebd050..029f65531 100755 --- a/docs.json +++ b/docs.json @@ -664,6 +664,16 @@ "quickstarts/autogen" ] }, + { + "group": "Agent Quickstarts", + "pages": [ + "agent-quickstart/node", + "agent-quickstart/python", + "agent-quickstart/rust", + "agent-quickstart/java", + "agent-quickstart/elixir" + ] + }, { "group": "LLM SDKs and Frameworks", "pages": [