diff --git a/agent-source-of-truth/curl.mdx b/agent-source-of-truth/curl.mdx index 6fded6451..11ee341a1 100644 --- a/agent-source-of-truth/curl.mdx +++ b/agent-source-of-truth/curl.mdx @@ -50,6 +50,8 @@ curl -X POST "https://api.firecrawl.dev/v2/search" \ "query": "site:docs.firecrawl.dev crawl webhooks", "sources": [{"type": "web"}, {"type": "news"}], "categories": [{"type": "research"}], + "includeDomains": ["docs.firecrawl.dev"], + "highlights": true, "limit": 10, "tbs": "qdr:m", "location": "San Francisco,California,United States", @@ -105,8 +107,18 @@ Successful responses include `success`, `data`, optional `warning`, `id`, and `c - `{ "type": "research" }` - `{ "type": "pdf" }` +- `includeDomains` + - Type: array of strings (hostnames) + - Use when: you want to restrict results to specific domains. + - Notes: cannot be combined with `excludeDomains`. + +- `excludeDomains` + - Type: array of strings (hostnames) + - Use when: you want to exclude results from specific domains. + - Notes: cannot be combined with `includeDomains`. + - `limit` - - Type: integer (minimum 1, maximum 100, default 5) + - Type: integer (minimum 1, maximum 100, default 10) - Use when: you want to cap results. - `tbs` @@ -121,6 +133,10 @@ Successful responses include `success`, `data`, optional `warning`, `id`, and `c - Type: string (default `"US"`) - Use when: you want ISO 3166-1 alpha-2 targeting (for example `"US"`). +- `highlights` + - Type: boolean (default true) + - Use when: you want query-relevant highlights for search results. + - `ignoreInvalidURLs` - Type: boolean (default false) - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -136,6 +152,11 @@ Successful responses include `success`, `data`, optional `warning`, `id`, and `c - `"zdr"`: end-to-end zero data retention - `"anon"`: anonymized zero data retention +- `threatProtection` + - Type: object + - Use when: you need per-request threat protection override (enterprise). + - Fields: `mode` (`"off"` | `"normal"`), `riskScoreThreshold` (0-100), `blacklist` (array, max 1000), `whitelist` (array, max 1000), `blockedTlds` (array, max 1000), `failurePolicy` (`"open"` | `"closed"`) + - `scrapeOptions` - Type: object - Use when: you want to scrape each search result (see Scrape parameters for fields). @@ -175,7 +196,9 @@ curl -X POST "https://api.firecrawl.dev/v2/scrape" \ "links", {"type": "json", "prompt": "Extract plan names and prices."}, {"type": "screenshot", "fullPage": true, "quality": 80, "viewport": {"width": 1280, "height": 720}}, - {"type": "changeTracking", "modes": ["git-diff"], "tag": "pricing"} + {"type": "changeTracking", "modes": ["git-diff"], "tag": "pricing"}, + {"type": "question", "question": "What are the pricing tiers?"}, + {"type": "highlights", "query": "enterprise pricing"} ], "headers": {"User-Agent": "FirecrawlDocsBot/1.0"}, "onlyMainContent": true, @@ -193,8 +216,7 @@ curl -X POST "https://api.firecrawl.dev/v2/scrape" \ "maxAge": 86400000, "minAge": 1, "storeInCache": true, - "profile": {"name": "docs-session", "saveChanges": true}, - "zeroDataRetention": false + "profile": {"name": "docs-session", "saveChanges": true} }' ``` @@ -203,10 +225,15 @@ curl -X POST "https://api.firecrawl.dev/v2/scrape" \ Successful responses include `success` and `data`. Common `data` fields (depending on `formats` and options): - `markdown`, `summary`, `html`, `rawHtml`, `screenshot`, `audio`, `video`, `links` +- `answer`: present when the `question` format is requested +- `highlights`: present when the `highlights` format is requested - `actions`: when the request included scrape-time `actions`, contains ordered results such as `screenshots`, `scrapes`, `javascriptReturns`, and `pdfs` - `metadata`: page metadata (`title`, `sourceURL`, `url`, `statusCode`, `error`, and other extracted fields) - `warning`: optional extraction or formatting notice - `changeTracking`: present when the `changeTracking` format is requested +- `branding`: brand design system data when `branding` format requested +- `product`: product data when `product` format requested +- `menu`: menu data when `menu` format requested ### Parameters @@ -228,13 +255,16 @@ Successful responses include `success` and `data`. Common `data` fields (dependi - `"changeTracking"`: change tracking output - `"json"`: JSON extraction - `"branding"`: branding profile output + - `"product"`: product data extraction + - `"menu"`: menu extraction - `"audio"`: audio extraction - `"video"`: video extraction - - Format object fields: - - `type`: one of the format strings above - - `prompt`, `schema`: JSON extraction options for `type: "json"` - - `modes`, `schema`, `prompt`, `tag`: change tracking options for `type: "changeTracking"` - - `fullPage`, `quality`, `viewport`: screenshot options for `type: "screenshot"` + - Object format types: + - `{"type": "json", "prompt": "...", "schema": {...}}`: JSON extraction + - `{"type": "question", "question": "..."}`: question-answer extraction. Answer in `data.answer`. + - `{"type": "highlights", "query": "..."}`: relevant source-text extraction. Result in `data.highlights`. + - `{"type": "screenshot", "fullPage": bool, "quality": int, "viewport": {...}}`: screenshot options + - `{"type": "changeTracking", "modes": [...], "schema": {...}, "prompt": "...", "tag": "..."}`: change tracking - `headers` - Type: object @@ -316,6 +346,25 @@ Successful responses include `success` and `data`. Common `data` fields (dependi - Type: boolean - Use when: you want Firecrawl to cache the result. +- `lockdown` + - Type: boolean + - Use when: you want to serve only previously cached results; never make outbound requests. + +- `redactPII` + - Type: boolean or object + - Use when: you want to redact personally identifiable information from returned content. + - Object fields: `mode` (`"accurate"` | `"aggressive"` | `"fast"`, default `"accurate"`), `entities` (array of `"PERSON"` | `"EMAIL"` | `"PHONE"` | `"LOCATION"` | `"FINANCIAL"` | `"SECRET"`), `replaceStyle` (`"tag"` | `"mask"` | `"remove"`, default `"tag"`) + +- `threatProtection` + - Type: object + - Use when: you need per-request threat protection override (enterprise). + - Fields: `mode` (`"off"` | `"normal"`), `riskScoreThreshold` (0-100), `blacklist` (array, max 1000), `whitelist` (array, max 1000), `blockedTlds` (array, max 1000), `failurePolicy` (`"open"` | `"closed"`) + +- `auditMetadata` + - Type: object + - Use when: you need user attribution for SIEM logging events. + - Fields: `username` (string, required, max 1024 chars) + - `profile` - Type: object with `name` and optional `saveChanges` - Use when: you want a persistent browser profile shared across scrapes and interactions. @@ -379,7 +428,7 @@ curl -X POST "https://api.firecrawl.dev/v2/scrape//interact" \ ### Response -Successful responses include `success` plus execution fields such as `stdout`, `result` (alias of stdout), `stderr`, `exitCode`, `killed`, and `error` (nullable). +Successful responses include `success` plus execution fields such as `stdout`, `result` (alias of stdout), `stderr`, `exitCode`, `killed`, `cdpUrl`, `liveViewUrl`, `interactiveLiveViewUrl`, and `error` (nullable). ### DELETE /scrape/{jobId}/interact @@ -489,4 +538,4 @@ curl -X POST "https://api.firecrawl.dev/v2/support/docs-search" \ - `firecrawl-docs/api-reference/v2-openapi.json` - `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` - `firecrawl/apps/python-sdk/firecrawl/v2/types.py` -- `firecrawl/apps/rust-sdk/src/v2/scrape.rs` +- `firecrawl/apps/rust-sdk/src/scrape.rs` diff --git a/agent-source-of-truth/elixir.mdx b/agent-source-of-truth/elixir.mdx index 703f6d6b2..30026d131 100644 --- a/agent-source-of-truth/elixir.mdx +++ b/agent-source-of-truth/elixir.mdx @@ -3,14 +3,14 @@ title: "Elixir Source of Truth" description: "Canonical Firecrawl Elixir source of truth for agents using key endpoints like search, scrape, and interact." --- -Canonical Firecrawl Elixir source of truth for agents. Generated from SDK source and the v2 OpenAPI spec. +Canonical Firecrawl Elixir source of truth for agents. Generated from SDK source (`:firecrawl` **v1.9.1**) and the v2 OpenAPI spec. ## Install Add to `mix.exs`: ```elixir -{:firecrawl, "~> 1.0.0"} +{:firecrawl, "~> 1.9"} ``` ## Authenticate @@ -52,6 +52,8 @@ Use search to discover relevant pages from a query, then pick URLs to scrape or query: "site:docs.firecrawl.dev crawl webhooks", sources: [:web, :news], categories: [:research], + include_domains: ["docs.firecrawl.dev"], + highlights: true, limit: 10, tbs: "qdr:m", location: "San Francisco,California,United States", @@ -97,6 +99,14 @@ Use search to discover relevant pages from a query, then pick URLs to scrape or - `"pdf"` or `:pdf` - `%{type: "github" | "research" | "pdf"}` +- `include_domains` + - Type: list of strings + - Use when: you want to restrict results to specific domains. + +- `exclude_domains` + - Type: list of strings + - Use when: you want to exclude results from specific domains. + - `limit` - Type: integer - Use when: you want to cap results. @@ -113,6 +123,11 @@ Use search to discover relevant pages from a query, then pick URLs to scrape or - Type: string - Use when: you want ISO 3166-1 alpha-2 targeting (for example `"US"`). +- `highlights` + - Type: boolean + - Use when: you want query-relevant highlights for search results. + - Notes: defaults to `true` server-side. + - `ignore_invalid_urls` - Type: boolean - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -161,7 +176,9 @@ Use scrape when you already have a URL and want structured content in one or mor "links", %{type: "json", prompt: "Extract plan names and prices."}, %{type: "screenshot", fullPage: true, quality: 80, viewport: %{width: 1280, height: 720}}, - %{type: "changeTracking", modes: ["git-diff"], tag: "pricing"} + %{type: "changeTracking", modes: ["git-diff"], tag: "pricing"}, + %{type: "question", question: "What are the pricing tiers?"}, + %{type: "highlights", query: "enterprise pricing"} ], headers: %{"User-Agent" => "FirecrawlDocsBot/1.0"}, only_main_content: true, @@ -179,8 +196,7 @@ Use scrape when you already have a URL and want structured content in one or mor max_age: 86400000, min_age: 1, store_in_cache: true, - profile: [name: "docs-session", save_changes: true], - zero_data_retention: false + profile: [name: "docs-session", save_changes: true] ) ``` @@ -204,8 +220,14 @@ Use scrape when you already have a URL and want structured content in one or mor - `"changeTracking"`: change tracking output - `"json"`: JSON extraction - `"branding"`: branding profile output + - `"product"`: product data extraction + - `"menu"`: menu extraction - `"audio"`: audio extraction - `"video"`: video extraction + - Format map types: + - `%{type: "json", prompt: "...", schema: %{...}}`: JSON extraction + - `%{type: "question", question: "..."}`: question-answer extraction + - `%{type: "highlights", query: "..."}`: relevant source-text extraction - Format map fields: - `type`: one of the format strings above - `prompt`, `schema`: JSON extraction options for `type: "json"` @@ -294,6 +316,18 @@ Use scrape when you already have a URL and want structured content in one or mor - Type: boolean - Use when: you want Firecrawl to cache the result. +- `lockdown` + - Type: boolean + - Use when: you want to serve only previously cached results; never make outbound requests. + +- `redact_pii` + - Type: boolean + - Use when: you want to redact personally identifiable information from returned content. + +- `audit_metadata` + - Type: keyword list with `username:` (required) + - Use when: you need user attribution for SIEM logging events. + - `profile` - Type: keyword list with `name:` and optional `save_changes:` (or `saveChanges:`) - Use when: you want a persistent browser profile shared across scrapes and interactions. @@ -341,7 +375,7 @@ Use interact when a page requires browser actions or code execution after a scra - Use when: you have a scrape job ID. - `code` - - Type: string + - Type: string (required) - Use when: you want to run code in the browser session. - `language` @@ -370,6 +404,7 @@ Use interact when a page requires browser actions or code execution after a scra - The Elixir client is OpenAPI-shaped; function names and parameter keys are generated from the spec. - Each public function has a bang (`!`) variant that raises on error instead of returning `{:error, _}`. - This SDK exposes code-based interactions only (no `prompt` parameter on `interact_with_scrape_browser_session`). +- All parameter validation uses `NimbleOptions`. ## Source Of Truth diff --git a/agent-source-of-truth/java.mdx b/agent-source-of-truth/java.mdx index 93da68332..a8a356038 100644 --- a/agent-source-of-truth/java.mdx +++ b/agent-source-of-truth/java.mdx @@ -3,7 +3,7 @@ title: "Java Source of Truth" description: "Canonical Firecrawl Java source of truth for agents using key endpoints like search, scrape, and interact." --- -Canonical Firecrawl Java source of truth for agents. Generated from SDK source and the v2 OpenAPI spec. +Canonical Firecrawl Java source of truth for agents. Generated from SDK source (`firecrawl-java` **v1.12.1**) and the v2 OpenAPI spec. ## Install @@ -13,14 +13,14 @@ Maven: com.firecrawl firecrawl-java - 1.2.0 + 1.12.1 ``` Gradle: ```gradle -implementation("com.firecrawl:firecrawl-java:1.2.0") +implementation("com.firecrawl:firecrawl-java:1.12.1") ``` ## Authenticate @@ -78,6 +78,8 @@ import com.firecrawl.models.LocationConfig; SearchOptions options = SearchOptions.builder() .sources(List.of("web", "news")) .categories(List.of("research")) + .includeDomains(List.of("docs.firecrawl.dev")) + .highlights(true) .limit(10) .tbs("qdr:m") .location("San Francisco,California,United States") @@ -124,6 +126,14 @@ SearchData results = client.search("site:docs.firecrawl.dev crawl webhooks", opt - `"pdf"`: PDF-focused results - `{type: "github" | "research" | "pdf"}`: typed category map form +- `options.includeDomains` + - Type: `List` + - Use when: you want to restrict results to specific domains. + +- `options.excludeDomains` + - Type: `List` + - Use when: you want to exclude results from specific domains. + - `options.limit` - Type: Integer - Use when: you want to cap results. @@ -136,6 +146,11 @@ SearchData results = client.search("site:docs.firecrawl.dev crawl webhooks", opt - Type: String - Use when: you want localized results. +- `options.highlights` + - Type: Boolean + - Use when: you want query-relevant highlights for search results. + - Notes: defaults to `true` server-side. + - `options.ignoreInvalidURLs` - Type: Boolean - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -148,10 +163,6 @@ SearchData results = client.search("site:docs.firecrawl.dev crawl webhooks", opt - Type: `ScrapeOptions` - Use when: you want to scrape each search result (see Scrape parameters for fields). -- `options.integration` - - Type: String - - Use when: the API expects an integration identifier on the request. - ## Scrape ### Why use it @@ -165,7 +176,7 @@ Use scrape when you already have a URL and want structured content in one or mor ### Return value -`scrape` returns `Document`. Typical getters include `getMarkdown()`, `getHtml()`, `getRawHtml()`, `getJson()`, `getMetadata()`, `getLinks()`, `getAudio()`, `getVideo()`, and additional fields when the corresponding formats are requested. +`scrape` returns `Document`. Typical getters include `getMarkdown()`, `getHtml()`, `getRawHtml()`, `getJson()`, `getMetadata()`, `getLinks()`, `getAudio()`, `getVideo()`, `getProduct()`, `getMenu()`, `getBranding()`, `getHighlights()`, `getAnswer()`, and additional fields when the corresponding formats are requested. ### Simple Example @@ -181,6 +192,8 @@ Document doc = client.scrape( ```java import com.firecrawl.models.ScrapeOptions; import com.firecrawl.models.JsonFormat; +import com.firecrawl.models.QuestionFormat; +import com.firecrawl.models.HighlightsFormat; List> actions = List.of( Map.of("type", "click", "selector", "#accept"), @@ -198,6 +211,8 @@ ScrapeOptions options = ScrapeOptions.builder() "markdown", "links", JsonFormat.builder().prompt("Extract plan names and prices.").build(), + QuestionFormat.builder().question("What are the pricing tiers?").build(), + HighlightsFormat.builder().query("enterprise pricing").build(), Map.of("type", "screenshot", "fullPage", true, "quality", 80) )) .onlyMainContent(true) @@ -222,7 +237,7 @@ Document doc = client.scrape("https://example.com/pricing", options); - Use when: you want to scrape a specific page. - `options.formats` - - Type: List of format strings or format maps + - Type: List of format strings or format objects - Use when: you want multiple output formats. - Confirmed format strings: - `"markdown"`: markdown content @@ -236,9 +251,15 @@ Document doc = client.scrape("https://example.com/pricing", options); - `"json"`: JSON extraction - `"attributes"`: attribute extraction - `"branding"`: branding profile output + - `"product"`: product data extraction + - `"menu"`: menu extraction - `"audio"`: audio extraction - `"video"`: video extraction - - Format object fields: + - Format object types: + - `JsonFormat.builder().prompt("...").schema(map).build()`: JSON extraction + - `QuestionFormat.builder().question("...").build()`: question-answer extraction. Answer returned in `document.getAnswer()`. + - `HighlightsFormat.builder().query("...").build()`: relevant source-text extraction. Result returned in `document.getHighlights()`. + - Format map fields: - `type`: one of the format strings above - `prompt`, `schema`: JSON extraction options for `type: "json"` - `modes`, `schema`, `prompt`, `tag`: change tracking options for `type: "changeTracking"` @@ -323,9 +344,18 @@ Document doc = client.scrape("https://example.com/pricing", options); - Type: Boolean - Use when: you want Firecrawl to cache the result. -- `options.integration` - - Type: String - - Use when: the API expects an integration identifier on the request. +- `options.lockdown` + - Type: Boolean + - Use when: you want to serve only previously cached results; never make outbound requests. + +- `options.redactPII` + - Type: Boolean + - Use when: you want to redact personally identifiable information from returned content. + +- `options.auditMetadata` + - Type: `AuditMetadata` + - Use when: you need user attribution for SIEM logging events. + - Fields: `username` (String, required) ## Interact @@ -417,6 +447,7 @@ BrowserDeleteResponse stopped = client.stopInteractiveBrowser(""); - Deprecated aliases: `scrapeExecute` → `interact`, `deleteScrapeBrowser` → `stopInteractiveBrowser` (and the corresponding `*Async` helpers). - The Java SDK exposes code-based interactions only: there is no `prompt` parameter on `interact` (unlike some other language SDKs). +- Async variants are available for all methods: `scrapeAsync`, `searchAsync`, `interactAsync`, `stopInteractiveBrowserAsync`. ## Source Of Truth @@ -426,7 +457,10 @@ BrowserDeleteResponse stopped = client.stopInteractiveBrowser(""); - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/SearchData.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/JsonFormat.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/QuestionFormat.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/HighlightsFormat.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/LocationConfig.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/AuditMetadata.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/BrowserExecuteResponse.java` - `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/BrowserDeleteResponse.java` diff --git a/agent-source-of-truth/node.mdx b/agent-source-of-truth/node.mdx index 3d261eb0b..8c8b68e2e 100644 --- a/agent-source-of-truth/node.mdx +++ b/agent-source-of-truth/node.mdx @@ -3,7 +3,7 @@ title: "Node.js Source of Truth" description: "Canonical Firecrawl Node.js source of truth for agents using key endpoints like search, scrape, and interact." --- -Canonical Firecrawl Node.js source of truth for agents. Aligned with `firecrawl` **v4.18.2** (`firecrawl/apps/js-sdk/firecrawl`) and the v2 OpenAPI spec. Method names, parameters, and types match the SDK public API. +Canonical Firecrawl Node.js source of truth for agents. Aligned with `firecrawl` **v4.32.0** (`firecrawl/apps/js-sdk/firecrawl`) and the v2 OpenAPI spec. Method names, parameters, and types match the SDK public API. ## Install @@ -55,6 +55,8 @@ const results = await client.search("site:docs.firecrawl.dev crawl webhooks", { limit: 10, tbs: "qdr:m", location: "San Francisco,California,United States", + includeDomains: ["docs.firecrawl.dev"], + highlights: true, ignoreInvalidURLs: true, timeout: 60000, scrapeOptions: { @@ -78,6 +80,7 @@ const results = await client.search("site:docs.firecrawl.dev crawl webhooks", { - `web`: web index hits - `news`: news hits - `images`: image hits +- `developer`: developer-focused hits **Wrong turn to avoid:** `search()` does not return `{ data: [...] }`. Do not access `result.data`. Web results are in `result.web`, news in `result.news`, images in `result.images`. @@ -107,7 +110,18 @@ const results = await client.search("site:docs.firecrawl.dev crawl webhooks", { - `"github"`: GitHub-focused results - `"research"`: research and academic results - `"pdf"`: PDF-focused results - - `{ type: "github" | "research" | "pdf" }`: typed category object form + - `"developer"`: developer-focused results + - `{ type: "github" | "research" | "pdf" | "developer" }`: typed category object form + +- `options.includeDomains` + - Type: array of strings + - Use when: you want to restrict results to specific domains. + - Notes: cannot be combined with `excludeDomains`. + +- `options.excludeDomains` + - Type: array of strings + - Use when: you want to exclude results from specific domains. + - Notes: cannot be combined with `includeDomains`. - `options.limit` - Type: number @@ -121,6 +135,11 @@ const results = await client.search("site:docs.firecrawl.dev crawl webhooks", { - Type: string - Use when: you want localized results. +- `options.highlights` + - Type: boolean + - Use when: you want query-relevant highlights for search results. + - Notes: defaults to `true` server-side. + - `options.ignoreInvalidURLs` - Type: boolean - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -129,6 +148,15 @@ const results = await client.search("site:docs.firecrawl.dev crawl webhooks", { - Type: number - Use when: you need a request timeout in milliseconds. +- `options.enterprise` + - Type: array of `"default" | "anon" | "zdr"` + - Use when: you need enterprise Zero Data Retention search. Must be enabled for your team. + +- `options.threatProtection` + - Type: `ThreatProtectionOptions` + - Use when: you need per-request threat protection override (enterprise). + - Fields: `mode` (`"off"` | `"normal"`), `riskScoreThreshold` (0-100), `blacklist`, `whitelist`, `blockedTlds`, `failurePolicy` (`"open"` | `"closed"`) + - `options.scrapeOptions` - Type: `ScrapeOptions` - Use when: you want to scrape each search result (see Scrape parameters for fields). The SDK runs the same validation as for `scrape` (for example plain string `"json"` in `formats` is rejected). @@ -161,7 +189,9 @@ const doc = await client.scrape("https://example.com/pricing", { { type: "json", prompt: "Extract plan names and prices." }, { type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 720 } }, { type: "changeTracking", modes: ["git-diff"], tag: "pricing" }, - { type: "attributes", selectors: [{ selector: "a", attribute: "href" }] } + { type: "attributes", selectors: [{ selector: "a", attribute: "href" }] }, + { type: "question", question: "What are the pricing tiers?" }, + { type: "highlights", query: "enterprise pricing" } ], headers: { "User-Agent": "FirecrawlDocsBot/1.0" @@ -206,12 +236,14 @@ const doc = await client.scrape("https://example.com/pricing", { - `"changeTracking"`: change tracking output (for options like `modes`, use `{ type: "changeTracking", modes: [...] }` — `modes` is required on that object in typings) - `"attributes"`: attribute extraction (use `{ type: "attributes", selectors: [...] }` when passing selectors) - `"branding"`: branding profile output + - `"product"`: product data extraction + - `"menu"`: menu extraction - `"audio"`: audio extraction - `"video"`: video extraction - Object-only format types (at minimum `type` as shown): - `{ type: "json", prompt?: string, schema?: JSON schema or Zod schema }`: at least one of `prompt` or `schema` is required (SDK validation). - - `{ type: "question", question: string }`: question-answer style extraction. - - `{ type: "highlights", query: string }`: relevant source-text extraction. + - `{ type: "question", question: string }`: question-answer style extraction. Answer returned in `document.answer`. + - `{ type: "highlights", query: string }`: relevant source-text extraction. Result returned in `document.highlights`. - `{ type: "screenshot", fullPage?, quality?, viewport? }`: same options as the string form but as an object. - `{ type: "changeTracking", modes: ("git-diff" | "json")[], schema?, prompt?, tag? }`: `modes` is required. - `{ type: "attributes", selectors: Array<{ selector, attribute }> }` @@ -312,6 +344,24 @@ const doc = await client.scrape("https://example.com/pricing", { - Type: boolean - Use when: you want Firecrawl to cache the result. +- `options.lockdown` + - Type: boolean + - Use when: you want to serve only previously cached results; never make outbound requests. Returns 404 on cache miss. + +- `options.redactPII` + - Type: boolean or `RedactPIIOptions` + - Use when: you want to redact personally identifiable information from returned content. + - Object fields: `mode` (`"accurate"` | `"aggressive"` | `"fast"`, default `"accurate"`), `entities` (array of `"PERSON"` | `"EMAIL"` | `"PHONE"` | `"LOCATION"` | `"FINANCIAL"` | `"SECRET"`), `replaceStyle` (`"tag"` | `"mask"` | `"remove"`, default `"tag"`) + +- `options.threatProtection` + - Type: `ThreatProtectionOptions` + - Use when: you need per-request threat protection override (enterprise). + - Fields: `mode` (`"off"` | `"normal"`), `riskScoreThreshold` (0-100), `blacklist`, `whitelist`, `blockedTlds`, `failurePolicy` (`"open"` | `"closed"`) + +- `options.auditMetadata` + - Type: `{ username: string }` + - Use when: you need user attribution for SIEM logging events. + - `options.profile` - Type: object with `name` and optional `saveChanges` - Use when: you want a persistent browser profile shared across scrapes and interactions. @@ -385,6 +435,7 @@ const result = await client.interact("", { - `stderr`: string - `exitCode`: number - `killed`: boolean +- `cdpUrl`: string (raw CDP WebSocket URL for direct Playwright/Puppeteer connection) - `liveViewUrl`: string - `interactiveLiveViewUrl`: string - `error`: string @@ -526,7 +577,7 @@ console.log(result.answer); ## Notes -- Deprecated client aliases: `scrapeExecute` → `interact`; `stopInteractiveBrowser` and `deleteScrapeBrowser` → `stopInteraction`. +- Deprecated client aliases: `scrapeUrl` → `scrape`; `scrapeExecute` → `interact`; `stopInteractiveBrowser` and `deleteScrapeBrowser` → `stopInteraction`. - The default `Firecrawl` export is the v2 client; v1 remains under `client.v1`. - Zod schemas passed to `formats` (for `json` or `changeTracking`) are converted to JSON Schema by the SDK. - The package declares **Node.js >= 22** in `engines`. @@ -537,7 +588,4 @@ console.log(result.answer); - `firecrawl/apps/js-sdk/firecrawl/src/index.ts` - `firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` - `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` -- `firecrawl/apps/js-sdk/firecrawl/src/v2/utils/validation.ts` -- `firecrawl/apps/js-sdk/firecrawl/src/v2/methods/search.ts` -- `firecrawl/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts` - `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-source-of-truth/python.mdx b/agent-source-of-truth/python.mdx index 41e594452..c4247b289 100644 --- a/agent-source-of-truth/python.mdx +++ b/agent-source-of-truth/python.mdx @@ -3,7 +3,7 @@ title: "Python Source of Truth" description: "Canonical Firecrawl Python source of truth for agents using key endpoints like search, scrape, and interact." --- -Canonical Firecrawl Python source of truth for agents. Generated from SDK source (`firecrawl-py` / `firecrawl` **4.22.1**) and the v2 OpenAPI spec. Method names, parameters, and return types match the v2 client in `firecrawl/v2/client.py` unless noted. +Canonical Firecrawl Python source of truth for agents. Generated from SDK source (`firecrawl-py` **4.32.0+**) and the v2 OpenAPI spec. Method names, parameters, and return types match the v2 client in `firecrawl/v2/client.py` unless noted. ## Install @@ -46,6 +46,7 @@ Returns a `SearchData` model with optional lists: - `web` — web hits (`SearchResultWeb` or full `Document` when `scrape_options` hydrates content) - `news` — news hits (`SearchResultNews` or `Document`) - `images` — image hits (`SearchResultImages` or `Document`) +- `developer` — developer-focused hits (`SearchResultWeb` or `Document`) Omitted buckets are `None` when the API did not return that key. @@ -71,6 +72,8 @@ results = client.search( limit=10, tbs="qdr:m", location="San Francisco,California,United States", + include_domains=["docs.firecrawl.dev"], + highlights=True, ignore_invalid_urls=True, timeout=300000, scrape_options=ScrapeOptions( @@ -83,11 +86,6 @@ results = client.search( include_tags=["main", "article"], exclude_tags=["nav", "footer"], wait_for=1000, - actions=[ - {"type": "click", "selector": "button#accept"}, - {"type": "wait", "milliseconds": 750}, - {"type": "scrape"}, - ], ), ) ``` @@ -115,12 +113,23 @@ results = client.search( - `"github"`: GitHub-focused results - `"research"`: research and academic results - `"pdf"`: PDF-focused results - - `Category(type="github" | "research" | "pdf")`: typed category object form + - `"developer"`: developer-focused results + - `Category(type="github" | "research" | "pdf" | "developer")`: typed category object form + +- `include_domains` + - Type: list of str + - Use when: you want to restrict results to specific domains. + - Notes: cannot be combined with `exclude_domains`. + +- `exclude_domains` + - Type: list of str + - Use when: you want to exclude results from specific domains. + - Notes: cannot be combined with `include_domains`. - `limit` - Type: int - Use when: you want to cap results. - - Notes: defaults to `5` in the SDK model. + - Notes: defaults to `5` in the SDK model. Max 100. - `tbs` - Type: str @@ -130,6 +139,11 @@ results = client.search( - Type: str - Use when: you want localized results. +- `highlights` + - Type: bool + - Use when: you want query-relevant highlights for search results. + - Notes: defaults to `True` server-side. + - `ignore_invalid_urls` - Type: bool - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -139,6 +153,15 @@ results = client.search( - Use when: you need a request timeout in milliseconds. - Notes: defaults to `300000` in the SDK model. +- `enterprise` + - Type: list of str + - Use when: you need enterprise Zero Data Retention search. Use `["zdr"]` for end-to-end or `["anon"]` for anonymized. Must be enabled for your team. + +- `threat_protection` + - Type: `ThreatProtectionOptions` + - Use when: you need per-request threat protection override (enterprise). + - Fields: `mode` (`"off"` | `"normal"`), `risk_score_threshold` (0-100), `blacklist`, `whitelist`, `blocked_tlds`, `failure_policy` (`"open"` | `"closed"`) + - `scrape_options` - Type: `ScrapeOptions` - Use when: you want to scrape each search result (see Scrape parameters for fields). @@ -171,6 +194,8 @@ doc = client.scrape( {"type": "screenshot", "full_page": True, "quality": 80, "viewport": {"width": 1280, "height": 720}}, {"type": "changeTracking", "modes": ["git-diff"], "tag": "pricing"}, {"type": "attributes", "selectors": [{"selector": "a", "attribute": "href"}]}, + {"type": "question", "question": "What are the pricing tiers?"}, + {"type": "highlights", "query": "enterprise pricing"}, ], headers={"User-Agent": "FirecrawlDocsBot/1.0"}, only_main_content=True, @@ -212,12 +237,14 @@ doc = client.scrape( - `"changeTracking"` or `"change_tracking"`: change tracking output - `"attributes"`: attribute extraction - `"branding"`: branding profile output + - `"product"`: product data extraction + - `"menu"`: menu extraction - `"audio"`: audio extraction - `"video"`: video extraction - Object-only format types: - `{"type": "json", ...}`: JSON extraction. Use an object, not the plain string `"json"`. - - `{"type": "question", "question": "..."}`: question-answer output. - - `{"type": "highlights", "query": "..."}`: relevant source-text output. + - `{"type": "question", "question": "..."}`: question-answer output. Answer returned in `document.answer`. + - `{"type": "highlights", "query": "..."}`: relevant source-text output. Result returned in `document.highlights`. - Format object fields: - `type`: one of the format strings above, or `"json"`, `"question"`, or `"highlights"` for object-only formats - `question`: for `type: "question"` @@ -316,6 +343,25 @@ doc = client.scrape( - Type: bool - Use when: you want Firecrawl to cache the result. +- `lockdown` + - Type: bool + - Use when: you want to serve only previously cached results; never make outbound requests. Returns 404 on cache miss. + +- `redact_pii` + - Type: bool or `RedactPIIOptions` + - Use when: you want to redact personally identifiable information from returned content. + - Notes: only available on `ScrapeOptions` (not as a direct kwarg on `client.scrape`). Pass `True` for defaults, or a `RedactPIIOptions` object with `mode` (`"accurate"` | `"aggressive"` | `"fast"`), `entities` (list of `"PERSON"` | `"EMAIL"` | `"PHONE"` | `"LOCATION"` | `"FINANCIAL"` | `"SECRET"`), `replace_style` (`"tag"` | `"mask"` | `"remove"`). + +- `threat_protection` + - Type: `ThreatProtectionOptions` + - Use when: you need per-request threat protection override (enterprise). + - Notes: only available on `ScrapeOptions` (not as a direct kwarg on `client.scrape`). + +- `audit_metadata` + - Type: `AuditMetadata` + - Use when: you need user attribution for SIEM logging events. + - Fields: `username` (str, required, max 1024 chars) + - `profile` - Type: dict with `name` and optional `save_changes` or `saveChanges` - Use when: you want a persistent browser profile shared across scrapes and interactions. @@ -384,7 +430,7 @@ result = client.interact( ### Return value -Returns `BrowserExecuteResponse`: `success`, optional `live_view_url`, `interactive_live_view_url`, `output`, `stdout`, `result`, `stderr`, `exit_code`, `killed`, `error` (API camelCase is normalized to snake_case on the model). +Returns `BrowserExecuteResponse`: `success`, optional `cdp_url`, `live_view_url`, `interactive_live_view_url`, `output`, `stdout`, `result`, `stderr`, `exit_code`, `killed`, `error` (API camelCase is normalized to snake_case on the model). ## Ask (Agentic Debugging) @@ -507,8 +553,9 @@ print(response.json()["answer"]) ## Notes -- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser` and `delete_scrape_browser` → `stop_interaction`. +- Deprecated aliases: `scrape_url` → `scrape`; `scrape_execute` → `interact`; `stop_interactive_browser` and `delete_scrape_browser` → `stop_interaction`. - The top-level `Firecrawl` client exposes v2 methods directly; v1 remains under `client.v1`. +- `FirecrawlApp` is an alias for `Firecrawl`; `AsyncFirecrawl` (alias `AsyncFirecrawlApp`) for async usage. - The bundled v2 OpenAPI snippet for `POST /v2/scrape/{jobId}/interact` may only document `code`; the Python SDK and server accept either `code` or `prompt` for this endpoint. ## Source Of Truth @@ -520,5 +567,4 @@ print(response.json()["answer"]) - `firecrawl/apps/python-sdk/firecrawl/v2/types.py` - `firecrawl/apps/python-sdk/firecrawl/v2/methods/search.py` - `firecrawl/apps/python-sdk/firecrawl/v2/methods/scrape.py` -- `firecrawl/apps/python-sdk/firecrawl/v2/utils/validation.py` - `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-source-of-truth/rust.mdx b/agent-source-of-truth/rust.mdx index a64c654ab..e83e7ad5a 100644 --- a/agent-source-of-truth/rust.mdx +++ b/agent-source-of-truth/rust.mdx @@ -3,7 +3,7 @@ title: "Rust Source of Truth" description: "Canonical Firecrawl Rust source of truth for agents using key endpoints like search, scrape, and interact." --- -Canonical Firecrawl Rust source of truth for agents. Generated from SDK source and the v2 OpenAPI spec. +Canonical Firecrawl Rust source of truth for agents. Generated from SDK source (`firecrawl` crate **v2.12.1**) and the v2 OpenAPI spec. ## Install @@ -11,7 +11,7 @@ Canonical Firecrawl Rust source of truth for agents. Generated from SDK source a cargo add firecrawl ``` -Crate: **`firecrawl`** on crates.io. The current SDK version is **2.0.0** (verify the latest release on crates.io before pinning). +Crate: **`firecrawl`** on crates.io. The current SDK version is **2.12.1**. ## Authenticate @@ -61,6 +61,8 @@ use firecrawl::{ let options = SearchOptions { sources: Some(vec![SearchSource::Web, SearchSource::News]), categories: Some(vec![SearchCategory::Research]), + include_domains: Some(vec!["docs.firecrawl.dev".to_string()]), + highlights: Some(true), limit: Some(10), tbs: Some("qdr:m".to_string()), location: Some("San Francisco,California,United States".to_string()), @@ -113,6 +115,14 @@ let results = client - Use when: you want to filter results by category. - Confirmed values: `Github`, `Research`, `Pdf` +- `options.include_domains` + - Type: `Vec` + - Use when: you want to restrict results to specific domains. + +- `options.exclude_domains` + - Type: `Vec` + - Use when: you want to exclude results from specific domains. + - `options.limit` - Type: u32 - Use when: you want to cap results. @@ -125,6 +135,11 @@ let results = client - Type: String - Use when: you want localized results. +- `options.highlights` + - Type: bool + - Use when: you want query-relevant highlights for search results. + - Notes: defaults to `true` server-side. + - `options.ignore_invalid_urls` - Type: bool - Use when: you want to drop URLs that cannot be scraped by other endpoints. @@ -137,11 +152,6 @@ let results = client - Type: `ScrapeOptions` - Use when: you want to scrape each search result (see Scrape parameters for fields). -- `options.integration` - - Type: `Option` - - Use when: you need an integration identifier for server-side tracking. - - Notes: omit in agent-oriented examples unless your product intentionally sets it. - ## Scrape ### Why use it @@ -182,6 +192,8 @@ let doc = client Format::Screenshot, Format::ChangeTracking, Format::Attributes, + Format::Question(QuestionFormat { question: "What are the pricing tiers?".to_string() }), + Format::Highlights(HighlightsFormat { query: "enterprise pricing".to_string() }), ]), json_options: Some(JsonOptions { prompt: Some("Extract plan names and prices.".to_string()), @@ -203,6 +215,7 @@ let doc = client }]), parsers: Some(vec![ParserConfig::Pdf { parser_type: "pdf".to_string(), + mode: Some("auto".to_string()), max_pages: Some(5), }]), actions: Some(vec![ @@ -225,7 +238,8 @@ let doc = client - `options.formats` - Type: `Vec` - Use when: you want multiple output formats. - - Confirmed values: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Audio`, `Video` + - Simple variants: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Product`, `Menu`, `Audio`, `Video` + - Object variants: `Question(QuestionFormat)` (fields: `question`), `Highlights(HighlightsFormat)` (fields: `query`) - `options.headers` - Type: `HashMap` @@ -260,7 +274,7 @@ let doc = client - Use when: you need file parsing controls. - Confirmed values: - `ParserConfig::Simple("pdf".to_string())` - - `ParserConfig::Pdf { parser_type: "pdf", max_pages: Some(n) }` + - `ParserConfig::Pdf { parser_type: "pdf", mode: Some("auto"), max_pages: Some(n) }` - `options.actions` - Type: `Vec` @@ -314,16 +328,24 @@ let doc = client - Type: bool - Use when: you want Firecrawl to cache the result. +- `options.lockdown` + - Type: bool + - Use when: you want to serve only previously cached results; never make outbound requests. + +- `options.redact_pii` + - Type: bool + - Use when: you want to redact personally identifiable information from returned content. + +- `options.audit_metadata` + - Type: `AuditMetadata` + - Use when: you need user attribution for SIEM logging events. + - Fields: `username` (String, required) + - `options.profile` - Type: `ProfileConfig` - Use when: you want a persistent browser profile shared across scrapes and interactions. - Confirmed fields: `name`, `save_changes` -- `options.integration` - - Type: `Option` - - Use when: you need an integration identifier for server-side tracking. - - Notes: omit in agent-oriented examples unless your product intentionally sets it. - - `options.json_options` - Type: `JsonOptions` - Use when: you want to configure JSON extraction. @@ -421,11 +443,6 @@ let stopped = client.stop_interaction("").await?; - Type: u32 - Use when: you need an execution timeout in seconds. -- `options.origin` - - Type: `Option` - - Use when: you need an optional origin label for execution telemetry. - - Notes: omit in agent-oriented examples unless your product intentionally sets it. - At least one of `options.code` or `options.prompt` must be non-empty; otherwise the SDK returns `FirecrawlError::Misuse` before calling the API. ### Response types @@ -452,9 +469,4 @@ The v2 OpenAPI spec currently models the interact request body with `code` as re - `firecrawl/apps/rust-sdk/src/client.rs` - `firecrawl/apps/rust-sdk/src/scrape.rs` - `firecrawl/apps/rust-sdk/src/search.rs` -- `firecrawl/apps/rust-sdk/src/crawl.rs` -- `firecrawl/apps/rust-sdk/src/map.rs` -- `firecrawl/apps/rust-sdk/src/batch_scrape.rs` -- `firecrawl/apps/rust-sdk/src/agent.rs` -- `firecrawl/apps/rust-sdk/src/types.rs` - `firecrawl-docs/api-reference/v2-openapi.json`