From deada2c8eaf91447fde444d8f43a84a867e105b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 13:26:08 +0000 Subject: [PATCH] Add agent quickstart docs for all SDK languages One canonical quickstart file per SDK language (Node.js, Python, Rust, Java, Elixir) covering search, scrape, and interact endpoints. Each file includes install, auth, examples, and complete parameter tables generated from SDK source and the v2 OpenAPI spec. Co-Authored-By: Claude Opus 4.6 (1M context) Claude-Session: https://claude.ai/code/session_01RFKPxUWsrc2bzs8JPyhj48 --- agent-quickstart/elixir.mdx | 186 ++++++++++++++++++++++++++++++++ agent-quickstart/java.mdx | 204 +++++++++++++++++++++++++++++++++++ agent-quickstart/node.mdx | 190 ++++++++++++++++++++++++++++++++ agent-quickstart/python.mdx | 184 +++++++++++++++++++++++++++++++ agent-quickstart/rust.mdx | 208 ++++++++++++++++++++++++++++++++++++ 5 files changed, 972 insertions(+) create mode 100644 agent-quickstart/elixir.mdx create mode 100644 agent-quickstart/java.mdx create mode 100644 agent-quickstart/node.mdx create mode 100644 agent-quickstart/python.mdx create mode 100644 agent-quickstart/rust.mdx diff --git a/agent-quickstart/elixir.mdx b/agent-quickstart/elixir.mdx new file mode 100644 index 000000000..e30a46609 --- /dev/null +++ b/agent-quickstart/elixir.mdx @@ -0,0 +1,186 @@ +--- +title: "Elixir Agent Quickstart" +description: "Canonical Firecrawl Elixir quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Elixir Agent Quickstart + +Canonical quickstart for agents integrating Firecrawl via the Elixir SDK. Generated from SDK source (`firecrawl` hex package) and the v2 OpenAPI spec. Function names and parameters match the auto-generated OpenAPI client. + +## Install + +Add to `mix.exs`: + +```elixir +{:firecrawl, "~> 1.9"} +``` + +## Authenticate + +```elixir +# config/runtime.exs or config.exs +config :firecrawl, api_key: System.get_env("FIRECRAWL_API_KEY") + +# or pass api_key per call +{:ok, res} = Firecrawl.search_and_scrape( + [query: "site:docs.firecrawl.dev webhooks"], + api_key: "fc-your-api-key" +) +``` + +## When To Use What + +- **`search`**: use when you start with a query and need discovery. Returns categorized results from web, news, and image sources. +- **`scrape`**: use when you already have a URL and want page content in one or more formats (markdown, HTML, JSON extraction, screenshots, etc.). +- **`interact`**: use when the page needs clicks, forms, or post-scrape browser actions. Operates on a browser session created by a prior scrape. + +## Search + +### Why use it + +Discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:` in the query (e.g. `site:docs.firecrawl.dev webhooks`). + +### Preferred SDK method + +`Firecrawl.search_and_scrape(params \\ [], opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.search_and_scrape!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, res} = Firecrawl.search_and_scrape( + query: "site:docs.firecrawl.dev webhook retries", + sources: [:web], + limit: 5, + scrape_options: [ + formats: ["markdown"], + only_main_content: true + ] +) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | string (required) | Search query. Use `site:example.com` to scope to a domain. | +| `sources` | list of atoms, strings, or maps | Which source types: `:web`, `:news`, `:images`, or `%{type: "web"}`. | +| `categories` | list of atoms, strings, or maps | Category filters: `:github`, `:research`, `:pdf`, or `%{type: "github"}`. | +| `include_domains` | list of strings | Restrict results to these domains. | +| `exclude_domains` | list of strings | Exclude results from these domains. | +| `limit` | integer | Max number of results. | +| `tbs` | string | Time-based filter (e.g. `qdr:d`, `qdr:w`). | +| `location` | string | Location string for geo-targeted results. | +| `country` | string | ISO 3166-1 alpha-2 country code (e.g. `"US"`). | +| `ignore_invalid_urls` | boolean | Drop URLs that cannot be scraped. | +| `timeout` | integer | Request timeout in milliseconds. | +| `highlights` | boolean | Generate query-relevant highlights. Defaults to `true`. | +| `enterprise` | list of strings | Enterprise options: `"zdr"`, `"anon"`. | +| `scrape_options` | keyword list | Scrape each search result (see Scrape parameters). | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`Firecrawl.scrape_and_extract_from_url(params \\ [], opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.scrape_and_extract_from_url!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, res} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com/pricing", + formats: [ + "markdown", + %{type: "json", prompt: "Extract plan names and prices."} + ], + only_main_content: true +) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | string (required) | URL to scrape. | +| `formats` | list of strings or maps | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"branding"`, `"audio"`, `"video"`. Maps: `%{type: "json", prompt: ...}`, `%{type: "screenshot", fullPage: true}`, `%{type: "changeTracking", modes: ["git-diff"]}`. | +| `headers` | map | Custom HTTP headers. | +| `include_tags` | list of strings | Only include these HTML tags. | +| `exclude_tags` | list of strings | Exclude these HTML tags. | +| `only_main_content` | boolean | Strip nav, footer, and boilerplate. | +| `timeout` | integer | Timeout in milliseconds. Default 60000, range 1000–300000. | +| `wait_for` | integer | Wait for the page to render (milliseconds). | +| `mobile` | boolean | Use mobile viewport. | +| `parsers` | list of strings or maps | File parsing. E.g. `"pdf"` or `%{type: "pdf", mode: "auto", maxPages: 5}`. | +| `actions` | list of maps | Pre-scrape browser actions. Types: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `location` | keyword list | Geo/language-aware scraping. Keys: `country:`, `languages:`. | +| `skip_tls_verification` | boolean | Skip TLS verification. | +| `remove_base64_images` | boolean | Drop base64 images from markdown output. | +| `block_ads` | boolean | Block ads and cookie popups. | +| `proxy` | atom | Proxy mode: `:basic`, `:enhanced`, `:auto`. | +| `max_age` | integer | Max age (ms) of cached content to reuse. | +| `min_age` | integer | Min age (ms) of cached content. | +| `store_in_cache` | boolean | Store result in Firecrawl cache. | +| `profile` | keyword list | Persistent browser profile. Keys: `name:`, `save_changes:`. | +| `zero_data_retention` | boolean | Enable zero data retention. | +| `lockdown` | boolean | Only serve cached results, no outbound request. | +| `redact_pii` | boolean | Redact personally identifiable information. | + +## Interact + +### Why use it + +Control the browser session tied to a prior scrape job. Use for code execution in the browser runtime. + +### Preferred SDK method + +`Firecrawl.interact_with_scrape_browser_session(job_id, params \\ [], opts \\ [])` → `{:ok, %Req.Response{}}` or `{:error, exception}` + +Bang variant: `Firecrawl.interact_with_scrape_browser_session!(job_id, params, opts)` raises on error. + +### Example + +```elixir +{:ok, res} = Firecrawl.interact_with_scrape_browser_session( + "", + code: "console.log(await page.title());", + language: :node, + timeout: 60 +) + +# When done, stop the session: +{:ok, _} = Firecrawl.stop_interactive_scrape_browser_session("") +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | string (required, 1st arg) | Scrape job ID from the scrape response metadata. | +| `code` | string (required) | Code to execute in the browser session. | +| `language` | atom or string | Runtime: `:python`, `:node`, `:bash`. | +| `timeout` | integer | Execution timeout in seconds. | + +### Stop session + +`Firecrawl.stop_interactive_scrape_browser_session(job_id)` ends the browser session. + +## Notes + +- The Elixir client is **auto-generated from the OpenAPI spec**; function names are derived from operation IDs, not hand-written aliases. +- This SDK exposes **code-based interactions only**: there is no `prompt` parameter on `interact_with_scrape_browser_session`. +- Each function has a bang (`!`) variant that raises on error instead of returning `{:error, _}`. +- The SDK appends `"origin": "elixir-sdk@"` to every request body for telemetry. +- Uses `snake_case` parameter keys in Elixir, converted to `camelCase` JSON keys on the wire. + +## Source Of Truth + +- `firecrawl/apps/elixir-sdk/mix.exs` +- `firecrawl/apps/elixir-sdk/lib/firecrawl.ex` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/java.mdx b/agent-quickstart/java.mdx new file mode 100644 index 000000000..b4fa617bf --- /dev/null +++ b/agent-quickstart/java.mdx @@ -0,0 +1,204 @@ +--- +title: "Java Agent Quickstart" +description: "Canonical Firecrawl Java quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Java Agent Quickstart + +Canonical quickstart for agents integrating Firecrawl via the Java SDK. Generated from SDK source (`firecrawl-java`) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +Maven: + +```xml + + com.firecrawl + firecrawl-java + 1.12.1 + +``` + +Gradle: + +```gradle +implementation("com.firecrawl:firecrawl-java:1.12.1") +``` + +## Authenticate + +```java +import com.firecrawl.client.FirecrawlClient; + +FirecrawlClient client = FirecrawlClient.builder() + .apiKey(System.getenv("FIRECRAWL_API_KEY")) + .build(); +``` + +## When To Use What + +- **`search`**: use when you start with a query and need discovery. Returns categorized results from web, news, and image sources. +- **`scrape`**: use when you already have a URL and want page content in one or more formats (markdown, HTML, JSON extraction, screenshots, etc.). +- **`interact`**: use when the page needs clicks, forms, or post-scrape browser actions. Operates on a browser session created by a prior scrape. + +## Search + +### Why use it + +Discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:` in the query (e.g. `site:docs.firecrawl.dev webhooks`). + +### Preferred SDK method + +- `client.search(query)` → `SearchData` +- `client.search(query, options)` → `SearchData` + +### Example + +```java +import com.firecrawl.models.SearchData; +import com.firecrawl.models.SearchOptions; +import com.firecrawl.models.ScrapeOptions; + +SearchOptions options = SearchOptions.builder() + .sources(List.of("web")) + .limit(5) + .scrapeOptions( + ScrapeOptions.builder() + .formats(List.of("markdown")) + .onlyMainContent(true) + .build() + ) + .build(); + +SearchData results = client.search("site:docs.firecrawl.dev webhook retries", options); +List> web = results.getWeb(); +``` + +Read result buckets with `getWeb()`, `getNews()`, and `getImages()`. Do not treat `SearchData` as a directly iterable list. + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | String (required, 1st arg) | Search query. Use `site:example.com` to scope to a domain. | +| `options.sources` | `List` | Which source types: `"web"`, `"news"`, `"images"`, or `{type: ...}` maps. | +| `options.categories` | `List` | Category filters: `"github"`, `"research"`, `"pdf"`, or `{type: ...}` maps. | +| `options.includeDomains` | `List` | Restrict results to these domains. Cannot combine with `excludeDomains`. | +| `options.excludeDomains` | `List` | Exclude results from these domains. Cannot combine with `includeDomains`. | +| `options.limit` | Integer | Max number of results. | +| `options.tbs` | String | Time-based filter (e.g. `qdr:d`, `qdr:w`). | +| `options.location` | String | Location string for geo-targeted results. | +| `options.ignoreInvalidURLs` | Boolean | Drop URLs that cannot be scraped. | +| `options.timeout` | Integer | Request timeout in milliseconds. | +| `options.highlights` | Boolean | Generate query-relevant highlights for results. Defaults to `true`. | +| `options.scrapeOptions` | `ScrapeOptions` | Scrape each search result (see Scrape parameters). | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +- `client.scrape(url)` → `Document` +- `client.scrape(url, options)` → `Document` + +### Example + +```java +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.JsonFormat; + +ScrapeOptions options = ScrapeOptions.builder() + .formats(List.of( + "markdown", + JsonFormat.builder().prompt("Extract plan names and prices.").build() + )) + .onlyMainContent(true) + .build(); + +Document doc = client.scrape("https://example.com/pricing", options); +System.out.println(doc.getMarkdown()); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | String (required, 1st arg) | URL to scrape. | +| `options.formats` | `List` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Objects: `JsonFormat.builder().prompt(...).schema(...).build()`, or maps for screenshot/changeTracking/attributes. | +| `options.headers` | `Map` | Custom HTTP headers. | +| `options.includeTags` | `List` | Only include these HTML tags. | +| `options.excludeTags` | `List` | Exclude these HTML tags. | +| `options.onlyMainContent` | Boolean | Strip nav, footer, and boilerplate. | +| `options.timeout` | Integer | Timeout in milliseconds. | +| `options.waitFor` | Integer | Wait for the page to render (milliseconds). | +| `options.mobile` | Boolean | Use mobile viewport. | +| `options.parsers` | `List` | File parsing. E.g. `"pdf"` or `{type: "pdf", maxPages: 5}`. | +| `options.actions` | `List>` | Pre-scrape browser actions. Types: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `options.location` | `LocationConfig` | Geo/language-aware scraping. | +| `options.skipTlsVerification` | Boolean | Skip TLS verification. | +| `options.removeBase64Images` | Boolean | Drop base64 images from markdown. | +| `options.blockAds` | Boolean | Block ads and cookie popups. | +| `options.proxy` | String | Proxy mode: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or custom URL. | +| `options.maxAge` | Long | Max age (ms) of cached content to reuse. | +| `options.storeInCache` | Boolean | Store result in Firecrawl cache. | +| `options.lockdown` | Boolean | Only serve cached results, no outbound request. | +| `options.redactPII` | Boolean | Redact personally identifiable information. | +| `options.auditMetadata` | `AuditMetadata` | User attribution for SIEM logging. Has `username` field. | + +## Interact + +### Why use it + +Control the browser session tied to a prior scrape job. Use for code execution in the browser runtime. + +### Preferred SDK method + +- `client.interact(jobId, code)` — defaults to `"node"` language +- `client.interact(jobId, code, language, timeout)` — `timeout` is seconds (1–300), null for API default + +### Example + +```java +import com.firecrawl.models.BrowserExecuteResponse; + +BrowserExecuteResponse result = client.interact( + "", + "console.log(await page.title());", + "node", + 60 +); + +// When done, stop the session: +client.stopInteractiveBrowser(""); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | String (required) | Scrape job ID from the scrape response metadata. | +| `code` | String (required) | Code to execute in the browser session. | +| `language` | String | Runtime: `"python"`, `"node"`, `"bash"`. Defaults to `"node"`. | +| `timeout` | Integer | Execution timeout in seconds (1–300). Null uses API default. | + +### Stop session + +`client.stopInteractiveBrowser(jobId)` ends the browser session. Returns `BrowserDeleteResponse` with `isSuccess()`, `getSessionDurationMs()`, `getCreditsBilled()`, `getError()`. + +## Notes + +- The Java SDK exposes **code-based interactions only**: there is no `prompt` parameter on `interact` (unlike Node.js, Python, and Rust SDKs). +- Deprecated aliases: `scrapeExecute` → `interact`; `deleteScrapeBrowser` → `stopInteractiveBrowser`. +- Uses `camelCase` parameter names matching the JSON API contract. + +## Source Of Truth + +- `firecrawl/apps/java-sdk/build.gradle.kts` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/node.mdx b/agent-quickstart/node.mdx new file mode 100644 index 000000000..05bfdb012 --- /dev/null +++ b/agent-quickstart/node.mdx @@ -0,0 +1,190 @@ +--- +title: "Node.js Agent Quickstart" +description: "Canonical Firecrawl Node.js quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Node.js Agent Quickstart + +Canonical quickstart for agents integrating Firecrawl via the Node.js/TypeScript SDK. Generated from SDK source (`firecrawl`) and the v2 OpenAPI spec. Method names, parameters, and types match the SDK public API. + +## Install + +```bash +npm install firecrawl +``` + +## Authenticate + +```ts +import { Firecrawl } from "firecrawl"; + +const client = new Firecrawl({ + apiKey: process.env.FIRECRAWL_API_KEY, +}); +``` + +## When To Use What + +- **`search`**: use when you start with a query and need discovery. Returns categorized results from web, news, and image sources. +- **`scrape`**: use when you already have a URL and want page content in one or more formats (markdown, HTML, JSON extraction, screenshots, etc.). +- **`interact`**: use when the page needs clicks, forms, or post-scrape browser actions. Operates on a browser session created by a prior scrape via `metadata.scrapeId`. + +## Search + +### Why use it + +Discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:` in the query (e.g. `site:docs.firecrawl.dev webhooks`). + +### Preferred SDK method + +`client.search(query, options?)` → `Promise` + +### Example + +```ts +const results = await client.search("site:docs.firecrawl.dev webhook retries", { + sources: ["web"], + limit: 5, + scrapeOptions: { + formats: ["markdown"], + onlyMainContent: true, + }, +}); + +for (const item of results.web ?? []) { + console.log(item.url, item.title); +} +``` + +**Wrong turn to avoid:** `search()` does not return `{ data: [...] }`. Web results are in `result.web`, news in `result.news`, images in `result.images`. + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | string (required, 1st arg) | Search query. Use `site:example.com` to scope to a domain. | +| `sources` | `("web" \| "news" \| "images" \| { type: ... })[]` | Which source types to search. | +| `categories` | `("github" \| "research" \| "pdf" \| "developer" \| { type: ... })[]` | Category filters for results. | +| `includeDomains` | `string[]` | Restrict results to these domains. Cannot combine with `excludeDomains`. | +| `excludeDomains` | `string[]` | Exclude results from these domains. Cannot combine with `includeDomains`. | +| `limit` | number | Max number of results. | +| `tbs` | string | Time-based filter (e.g. `qdr:d` for past day, `qdr:w` for past week). | +| `location` | string | Location string for geo-targeted results. | +| `ignoreInvalidURLs` | boolean | Drop URLs that cannot be scraped by other endpoints. | +| `timeout` | number | Request timeout in milliseconds. | +| `highlights` | boolean | Generate query-relevant highlights for results. Defaults to `true`. | +| `scrapeOptions` | `ScrapeOptions` | Scrape each search result with these options (see Scrape parameters). | +| `enterprise` | `("default" \| "anon" \| "zdr")[]` | Enterprise options (zero data retention, anonymized search). | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, options?)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com/pricing", { + formats: [ + "markdown", + { type: "json", prompt: "Extract plan names and prices." }, + ], + onlyMainContent: true, +}); + +console.log(doc.markdown); +console.log(doc.json); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | string (required, 1st arg) | URL to scrape. | +| `formats` | `FormatOption[]` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Objects: `{ type: "json", prompt?, schema? }`, `{ type: "question", question }`, `{ type: "highlights", query }`, `{ type: "screenshot", fullPage?, quality?, viewport? }`, `{ type: "changeTracking", modes, schema?, prompt?, tag? }`, `{ type: "attributes", selectors: [{ selector, attribute }] }`. | +| `headers` | `Record` | Custom HTTP headers for the request. | +| `includeTags` | `string[]` | Only include these HTML tags in output. | +| `excludeTags` | `string[]` | Exclude these HTML tags from output. | +| `onlyMainContent` | boolean | Strip nav, footer, and boilerplate. | +| `timeout` | number | Timeout in milliseconds. | +| `waitFor` | number | Wait for the page to render (milliseconds). | +| `mobile` | boolean | Use mobile viewport. | +| `parsers` | `(string \| { type: "pdf", mode?: "fast" \| "auto" \| "ocr", maxPages?: number })[]` | File parsing controls. | +| `actions` | `ActionOption[]` | Pre-scrape browser actions. Types: `wait` (milliseconds/selector), `click` (selector), `write` (text), `press` (key), `scroll` (direction), `screenshot`, `scrape`, `executeJavascript` (script), `pdf`. | +| `location` | `{ country?: string, languages?: string[] }` | Geo/language-aware scraping. | +| `skipTlsVerification` | boolean | Skip TLS verification. | +| `removeBase64Images` | boolean | Drop base64 images from markdown output. | +| `fastMode` | boolean | Faster scrapes with reduced fidelity. | +| `blockAds` | boolean | Block ads and cookie popups. | +| `proxy` | `"basic" \| "stealth" \| "enhanced" \| "auto" \| string` | Proxy mode or custom proxy URL. | +| `maxAge` | number | Max age (ms) of cached content to reuse. `0` bypasses cache. | +| `minAge` | number | Min age (ms) of cached content. | +| `storeInCache` | boolean | Store result in Firecrawl cache. | +| `lockdown` | boolean | Only serve cached results, no outbound request. | +| `redactPII` | `boolean \| RedactPIIOptions` | Redact personally identifiable information. Pass `true` for defaults or an object with `mode`, `entities`, `replaceStyle`. | +| `auditMetadata` | `{ username: string }` | User attribution for SIEM logging. | +| `profile` | `{ name: string, saveChanges?: boolean }` | Persistent browser profile across scrapes/interactions. | + +## Interact + +### Why use it + +Control the browser session tied to a prior scrape job. Use for clicks, form fills, code execution, or natural-language browser instructions. Requires a `scrapeId` from a previous scrape's `metadata`. + +### Preferred SDK method + +`client.interact(jobId, args)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com", { + formats: ["markdown"], +}); +const jobId = doc.metadata?.scrapeId; +if (!jobId) throw new Error("Missing scrapeId"); + +const result = await client.interact(jobId, { + prompt: "Click the pricing tab and summarize the plans.", +}); +console.log(result.output); + +// When done, stop the session: +await client.stopInteraction(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | string (required, 1st arg) | Scrape job ID from `document.metadata.scrapeId`. | +| `code` | string | Code to execute in the browser session (e.g. Playwright `page` API). Either `code` or `prompt` required. | +| `prompt` | string | Natural-language instruction for the browser agent. Either `code` or `prompt` required. | +| `language` | `"python" \| "node" \| "bash"` | Runtime for code execution. Defaults to `"node"`. | +| `timeout` | number | Execution timeout in seconds. | + +### Stop session + +`client.stopInteraction(jobId)` ends the browser session. Returns `{ success, sessionDurationMs?, creditsBilled?, error? }`. + +## Notes + +- Deprecated aliases: `scrapeExecute` → `interact`; `stopInteractiveBrowser`, `deleteScrapeBrowser` → `stopInteraction`; `scrapeUrl` → `scrape`. +- The default `Firecrawl` export is the v2 client; v1 remains under `client.v1`. +- Zod schemas passed in `formats` (for `json` or `changeTracking`) are converted to JSON Schema by the SDK. +- The package requires **Node.js ≥ 22**. + +## Source Of Truth + +- `firecrawl/apps/js-sdk/firecrawl/package.json` +- `firecrawl/apps/js-sdk/firecrawl/src/index.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/methods/search.ts` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/python.mdx b/agent-quickstart/python.mdx new file mode 100644 index 000000000..9d59fd3e1 --- /dev/null +++ b/agent-quickstart/python.mdx @@ -0,0 +1,184 @@ +--- +title: "Python Agent Quickstart" +description: "Canonical Firecrawl Python quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Python Agent Quickstart + +Canonical quickstart for agents integrating Firecrawl via the Python SDK. Generated from SDK source (`firecrawl-py`) and the v2 OpenAPI spec. Method names, parameters, and types match the v2 client. + +## Install + +```bash +pip install firecrawl-py +``` + +## Authenticate + +```python +import os +from firecrawl import Firecrawl + +client = Firecrawl(api_key=os.environ.get("FIRECRAWL_API_KEY")) +``` + +## When To Use What + +- **`search`**: use when you start with a query and need discovery. Returns categorized results from web, news, and image sources. +- **`scrape`**: use when you already have a URL and want page content in one or more formats (markdown, HTML, JSON extraction, screenshots, etc.). +- **`interact`**: use when the page needs clicks, forms, or post-scrape browser actions. Operates on a browser session created by a prior scrape via `metadata.scrape_id`. + +## Search + +### Why use it + +Discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:` in the query (e.g. `site:docs.firecrawl.dev webhooks`). + +### Preferred SDK method + +`client.search(query, **options)` → `SearchData` + +### Example + +```python +results = client.search( + "site:docs.firecrawl.dev webhook retries", + sources=["web"], + limit=5, + scrape_options={"formats": ["markdown"], "only_main_content": True}, +) + +for item in results.web or []: + print(getattr(item, "url", None), getattr(item, "title", None)) +``` + +**Wrong turn to avoid:** `search()` does not return `{ data: [...] }`. Web results are in `result.web`, news in `result.news`, images in `result.images`. + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | str (required, 1st arg) | Search query. Use `site:example.com` to scope to a domain. | +| `sources` | list of `"web"`, `"news"`, `"images"`, or `Source` objects | Which source types to search. | +| `categories` | list of `"github"`, `"research"`, `"pdf"`, `"developer"`, or `Category` objects | Category filters for results. | +| `include_domains` | list of str | Restrict results to these domains. Cannot combine with `exclude_domains`. | +| `exclude_domains` | list of str | Exclude results from these domains. Cannot combine with `include_domains`. | +| `limit` | int | Max number of results. Defaults to `5` in the SDK model. | +| `tbs` | str | Time-based filter (e.g. `qdr:d` for past day, `qdr:w` for past week). | +| `location` | str | Location string for geo-targeted results. | +| `ignore_invalid_urls` | bool | Drop URLs that cannot be scraped by other endpoints. | +| `timeout` | int | Request timeout in milliseconds. Defaults to `300000` in the SDK model. | +| `highlights` | bool | Generate query-relevant highlights for results. Defaults to `true`. | +| `scrape_options` | `ScrapeOptions` or dict | Scrape each search result with these options (see Scrape parameters). | +| `enterprise` | list of str | Enterprise options: `["zdr"]` for zero data retention, `["anon"]` for anonymized search. | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, **options)` → `Document` + +### Example + +```python +doc = client.scrape( + "https://example.com/pricing", + formats=[ + "markdown", + {"type": "json", "prompt": "Extract plan names and prices."}, + ], + only_main_content=True, +) + +print(doc.markdown) +print(doc.json) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | str (required, 1st arg) | URL to scrape. | +| `formats` | list of format strings or dicts | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"` / `"raw_html"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"` / `"change_tracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Objects: `{"type": "json", "prompt": ..., "schema": ...}`, `{"type": "question", "question": ...}`, `{"type": "highlights", "query": ...}`, `{"type": "screenshot", "full_page": ..., "quality": ..., "viewport": ...}`, `{"type": "changeTracking", "modes": [...], "tag": ...}`, `{"type": "attributes", "selectors": [...]}`. | +| `headers` | dict | Custom HTTP headers for the request. | +| `include_tags` | list of str | Only include these HTML tags in output. | +| `exclude_tags` | list of str | Exclude these HTML tags from output. | +| `only_main_content` | bool | Strip nav, footer, and boilerplate. | +| `timeout` | int | Timeout in milliseconds. | +| `wait_for` | int | Wait for the page to render (milliseconds). | +| `mobile` | bool | Use mobile viewport. | +| `parsers` | list of str or dicts | File parsing controls. E.g. `"pdf"` or `{"type": "pdf", "mode": "auto", "max_pages": 5}`. | +| `actions` | list of dicts | Pre-scrape browser actions. Types: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `location` | dict | Geo/language-aware scraping. Keys: `country`, `languages`. | +| `skip_tls_verification` | bool | Skip TLS verification. | +| `remove_base64_images` | bool | Drop base64 images from markdown output. | +| `fast_mode` | bool | Faster scrapes with reduced fidelity. | +| `block_ads` | bool | Block ads and cookie popups. | +| `proxy` | str | Proxy mode: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. | +| `max_age` | int | Max age (ms) of cached content to reuse. | +| `min_age` | int | Min age (ms) of cached content. Only available on `scrape_options` in `search`, not on `scrape` directly. | +| `store_in_cache` | bool | Store result in Firecrawl cache. | +| `lockdown` | bool | Only serve cached results, no outbound request. | +| `profile` | dict | Persistent browser profile. Keys: `name`, `save_changes` or `saveChanges`. | +| `audit_metadata` | `AuditMetadata` or dict | User attribution for SIEM logging. Has `username` field. | + +## Interact + +### Why use it + +Control the browser session tied to a prior scrape job. Use for clicks, form fills, code execution, or natural-language browser instructions. Requires a `scrape_id` from a previous scrape's `metadata`. + +### Preferred SDK method + +`client.interact(job_id, code=None, *, prompt=None, language="node", timeout=None)` + +`prompt` is keyword-only. At least one of `code` or `prompt` must be non-empty. + +### Example + +```python +doc = client.scrape("https://example.com", formats=["markdown"]) +job_id = doc.metadata.scrape_id if doc.metadata else None +if not job_id: + raise RuntimeError("Missing scrape_id") + +result = client.interact(job_id, prompt="Click the pricing tab and summarize the plans.") +print(result.output) + +# When done, stop the session: +client.stop_interaction(job_id) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | str (required, 1st arg) | Scrape job ID from `document.metadata.scrape_id`. | +| `code` | str | Code to execute in the browser session. Either `code` or `prompt` required. | +| `prompt` | str (keyword-only) | Natural-language instruction for the browser agent. Either `code` or `prompt` required. | +| `language` | str | Runtime for code execution: `"python"`, `"node"`, `"bash"`. Defaults to `"node"`. | +| `timeout` | int | Execution timeout in seconds. | + +### Stop session + +`client.stop_interaction(job_id)` ends the browser session. Returns `BrowserDeleteResponse` with `success`, optional `session_duration_ms`, `credits_billed`, `error`. + +## Notes + +- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser`, `delete_scrape_browser` → `stop_interaction`. +- The top-level `Firecrawl` client exposes v2 methods directly; v1 remains under `client.v1`. +- Python uses `snake_case` parameter names. The SDK accepts both `snake_case` and `camelCase` for format type strings (e.g. `"rawHtml"` or `"raw_html"`). + +## Source Of Truth + +- `firecrawl/apps/python-sdk/pyproject.toml` +- `firecrawl/apps/python-sdk/firecrawl/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/types.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/methods/scrape.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/methods/search.py` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/rust.mdx b/agent-quickstart/rust.mdx new file mode 100644 index 000000000..343ad5e77 --- /dev/null +++ b/agent-quickstart/rust.mdx @@ -0,0 +1,208 @@ +--- +title: "Rust Agent Quickstart" +description: "Canonical Firecrawl Rust quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Rust Agent Quickstart + +Canonical quickstart for agents integrating Firecrawl via the Rust SDK. Generated from SDK source (`firecrawl` crate) and the v2 OpenAPI spec. Method names, parameters, and types match the SDK public API. + +## Install + +```bash +cargo add firecrawl +``` + +## Authenticate + +```rust +use firecrawl::Client; + +let client = Client::new("fc-your-api-key")?; +// Self-hosted: +// let client = Client::new_selfhosted("http://localhost:3002", Some("fc-your-api-key"))?; +``` + +## When To Use What + +- **`search`**: use when you start with a query and need discovery. Returns categorized results from web, news, and image sources. +- **`scrape`**: use when you already have a URL and want page content in one or more formats (markdown, HTML, JSON extraction, screenshots, etc.). +- **`interact`**: use when the page needs clicks, forms, or post-scrape browser actions. Operates on a browser session created by a prior scrape. + +## Search + +### Why use it + +Discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:` in the query (e.g. `site:docs.firecrawl.dev webhooks`). + +### Preferred SDK method + +`client.search(query, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, SearchOptions, ScrapeOptions, Format}; + +let options = SearchOptions { + sources: Some(vec![SearchSource::Web]), + limit: Some(5), + scrape_options: Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + only_main_content: Some(true), + ..Default::default() + }), + ..Default::default() +}; + +let results = client + .search("site:docs.firecrawl.dev webhook retries", options) + .await?; +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `impl AsRef` (required) | Search query. Use `site:example.com` to scope to a domain. | +| `options.sources` | `Vec` | Which source types to search. Values: `Web`, `News`, `Images`. | +| `options.categories` | `Vec` | Category filters. Values: `Github`, `Research`, `Pdf`. | +| `options.include_domains` | `Vec` | Restrict results to these domains. Cannot combine with `exclude_domains`. | +| `options.exclude_domains` | `Vec` | Exclude results from these domains. Cannot combine with `include_domains`. | +| `options.limit` | `u32` | Max number of results. Default: 5, max: 20. | +| `options.tbs` | `String` | Time-based filter (e.g. `qdr:d`, `qdr:w`). | +| `options.location` | `String` | Location string for geo-targeted results. | +| `options.ignore_invalid_urls` | `bool` | Drop URLs that cannot be scraped. | +| `options.timeout` | `u32` | Request timeout in milliseconds. | +| `options.highlights` | `bool` | Generate query-relevant highlights. Defaults to `true`. | +| `options.scrape_options` | `ScrapeOptions` | Scrape each search result (see Scrape parameters). | + +## Scrape + +### Why use it + +Get structured content from a URL in one or more formats: markdown, HTML, JSON extraction, screenshots, and more. + +### Preferred SDK method + +`client.scrape(url, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format, JsonOptions}; + +let doc = client + .scrape("https://example.com/pricing", ScrapeOptions { + formats: Some(vec![Format::Markdown, Format::Json]), + json_options: Some(JsonOptions { + prompt: Some("Extract plan names and prices.".to_string()), + ..Default::default() + }), + only_main_content: Some(true), + ..Default::default() + }) + .await?; +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `impl AsRef` (required) | URL to scrape. | +| `options.formats` | `Vec` | Output formats. Values: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Audio`, `Video`. | +| `options.headers` | `HashMap` | Custom HTTP headers. | +| `options.include_tags` | `Vec` | Only include these HTML tags. | +| `options.exclude_tags` | `Vec` | Exclude these HTML tags. | +| `options.only_main_content` | `bool` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `u32` | Timeout in milliseconds. | +| `options.wait_for` | `u32` | Wait for the page to render (milliseconds). | +| `options.mobile` | `bool` | Use mobile viewport. | +| `options.parsers` | `Vec` | File parsing. E.g. `ParserConfig::Pdf { parser_type: "pdf", max_pages: Some(5) }`. | +| `options.actions` | `Vec` | Pre-scrape browser actions. Types: `Wait`, `Click`, `Write`, `Press`, `Scroll`, `Screenshot`, `Scrape`, `ExecuteJavascript`, `Pdf`. | +| `options.location` | `LocationConfig` | Geo/language-aware scraping. Fields: `country`, `languages`. | +| `options.skip_tls_verification` | `bool` | Skip TLS verification. | +| `options.remove_base64_images` | `bool` | Drop base64 images from markdown output. | +| `options.fast_mode` | `bool` | Faster scrapes with reduced fidelity. | +| `options.block_ads` | `bool` | Block ads and cookie popups. | +| `options.proxy` | `ProxyType` | Proxy mode. Values: `Basic`, `Stealth`, `Enhanced`, `Auto`. | +| `options.max_age` | `u32` | Max age (ms) of cached content to reuse. | +| `options.min_age` | `u32` | Min age (ms) of cached content. | +| `options.store_in_cache` | `bool` | Store result in Firecrawl cache. | +| `options.lockdown` | `bool` | Only serve cached results, no outbound request. | +| `options.redact_pii` | `bool` | Redact personally identifiable information. | +| `options.audit_metadata` | `AuditMetadata` | User attribution for SIEM logging. Has `username` field. | +| `options.profile` | `ProfileConfig` | Persistent browser profile. Fields: `name`, `save_changes`. | +| `options.json_options` | `JsonOptions` | JSON extraction config. Fields: `schema`, `system_prompt`, `prompt`. | +| `options.screenshot_options` | `ScreenshotOptions` | Screenshot config. Fields: `full_page`, `quality`, `viewport`. | +| `options.change_tracking_options` | `ChangeTrackingOptions` | Change tracking config. Fields: `modes` (`GitDiff`/`Json`), `schema`, `prompt`, `tag`. | +| `options.attribute_selectors` | `Vec` | Attribute extraction. Fields: `selector`, `attribute`. | + +## Interact + +### Why use it + +Control the browser session tied to a prior scrape job. Use for clicks, form fills, code execution, or natural-language browser instructions. + +### Preferred SDK method + +`client.interact(job_id, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format, ScrapeExecuteOptions}; + +let doc = client + .scrape("https://example.com", ScrapeOptions { + formats: Some(vec![Format::Markdown]), + ..Default::default() + }) + .await?; + +let job_id = doc.metadata + .and_then(|m| m.scrape_id) + .expect("Missing scrapeId"); + +let result = client + .interact(&job_id, ScrapeExecuteOptions { + prompt: Some("Click the pricing tab and summarize the plans.".to_string()), + ..Default::default() + }) + .await?; + +// When done, stop the session: +client.stop_interaction(&job_id).await?; +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `impl AsRef` (required) | Scrape job ID from the scrape response metadata. | +| `options.code` | `Option` | Code to execute in the browser session. Either `code` or `prompt` required. | +| `options.prompt` | `Option` | Natural-language instruction for the browser agent. Either `code` or `prompt` required. | +| `options.language` | `ScrapeExecuteLanguage` | Runtime: `Python`, `Node`, `Bash`. Defaults to `Node`. | +| `options.timeout` | `u32` | Execution timeout in seconds. | + +At least one of `code` or `prompt` must be non-empty; otherwise the SDK returns `FirecrawlError::Misuse`. + +### Stop session + +`client.stop_interaction(job_id)` ends the browser session. Returns `ScrapeBrowserDeleteResponse` with `success`, optional `session_duration_ms`, `credits_billed`, `error`. + +## Notes + +- Deprecated aliases: `scrape_execute`, `stop_interactive_browser`, `delete_scrape_browser` map to `interact` and `stop_interaction`. +- `ScrapeOptions` uses dedicated structs for advanced format options: `json_options`, `screenshot_options`, `change_tracking_options`. +- `search_and_scrape(query, limit)` is a convenience helper that calls `search` with default `ScrapeOptions` and returns `Vec`. +- All types are exported at the crate root: `use firecrawl::Client` (not `use firecrawl::v2::Client`). + +## Source Of Truth + +- `firecrawl/apps/rust-sdk/Cargo.toml` +- `firecrawl/apps/rust-sdk/src/lib.rs` +- `firecrawl/apps/rust-sdk/src/client.rs` +- `firecrawl/apps/rust-sdk/src/scrape.rs` +- `firecrawl/apps/rust-sdk/src/search.rs` +- `firecrawl-docs/api-reference/v2-openapi.json`