From 08c8a47f0c30e5bcfa367aa044631c1800e2dd08 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 13:26:47 +0000 Subject: [PATCH] docs: add agent quickstart files for all SDK languages One quickstart per language (Node.js, Python, Rust, Java, Elixir) covering search, scrape, and interact endpoints with confirmed parameters from SDK source and OpenAPI spec. Co-Authored-By: Claude Opus 4.6 (1M context) Claude-Session: https://claude.ai/code/session_01N1ED7edNzHprSG9RYjZyy4 --- agent-quickstart/elixir.mdx | 192 ++++++++++++++++++++++++++++++ agent-quickstart/java.mdx | 227 ++++++++++++++++++++++++++++++++++++ agent-quickstart/node.mdx | 197 +++++++++++++++++++++++++++++++ agent-quickstart/python.mdx | 204 ++++++++++++++++++++++++++++++++ agent-quickstart/rust.mdx | 213 +++++++++++++++++++++++++++++++++ 5 files changed, 1033 insertions(+) create mode 100644 agent-quickstart/elixir.mdx create mode 100644 agent-quickstart/java.mdx create mode 100644 agent-quickstart/node.mdx create mode 100644 agent-quickstart/python.mdx create mode 100644 agent-quickstart/rust.mdx diff --git a/agent-quickstart/elixir.mdx b/agent-quickstart/elixir.mdx new file mode 100644 index 000000000..93244d8dd --- /dev/null +++ b/agent-quickstart/elixir.mdx @@ -0,0 +1,192 @@ +--- +title: "Elixir Agent Quickstart" +description: "Canonical Firecrawl Elixir quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Elixir Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` Hex package) and the v2 OpenAPI spec. Function names match the auto-generated client from the OpenAPI spec. + +## Install + +Add to `mix.exs`: + +```elixir +{:firecrawl, "~> 1.9"} +``` + +## Authenticate + +```elixir +# config/runtime.exs or config.exs +config :firecrawl, api_key: System.get_env("FIRECRAWL_API_KEY") + +# Or pass api_key per call: +{:ok, res} = Firecrawl.search_and_scrape( + [query: "site:docs.firecrawl.dev webhook retries"], + api_key: "fc-your-api-key" +) +``` + +Self-hosted: + +```elixir +{:ok, res} = Firecrawl.scrape_and_extract_from_url( + [url: "https://example.com"], + base_url: "http://localhost:3002/v2", + api_key: "fc-your-api-key" +) +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`Firecrawl.search_and_scrape(params \\ [], opts \\ [])` + +### Example + +```elixir +{:ok, res} = Firecrawl.search_and_scrape( + query: "site:docs.firecrawl.dev webhook retries", + sources: [:web], + limit: 5, + scrape_options: [ + formats: ["markdown"], + only_main_content: true + ] +) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | string | Search query. Use `site:example.com` to limit to a domain. Required. | +| `sources` | list | Sources: `:web`, `:news`, `:images`. Default: `[:web]`. | +| `categories` | list | Filter by category. | +| `limit` | integer | Cap the number of results. | +| `tbs` | string | Time-based filter (e.g. `"qdr:d"`, `"qdr:w"`). | +| `location` | string | Localized results. | +| `country` | string | ISO country code for geo-targeting (e.g. `"US"`). | +| `ignore_invalid_urls` | boolean | Drop URLs that cannot be scraped. | +| `timeout` | integer | Request timeout in milliseconds. | +| `scrape_options` | keyword list | Scrape each search result. See Scrape parameters. | +| `include_domains` | list of strings | Only include results from these domains. | +| `exclude_domains` | list of strings | Exclude results from these domains. | +| `highlights` | boolean | Return highlights (defaults to true). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`Firecrawl.scrape_and_extract_from_url(params \\ [], opts \\ [])` + +### Example + +```elixir +{:ok, res} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com/pricing", + formats: ["markdown", "links", %{"type" => "json", "prompt" => "Extract plan names and prices."}], + only_main_content: true, + wait_for: 1000 +) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `url` | string | The page URL to scrape. Required. | +| `formats` | list | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Use maps for object forms. | +| `headers` | any | Custom request headers. | +| `include_tags` | list of strings | Include only specific HTML tags. | +| `exclude_tags` | list of strings | Exclude specific HTML tags. | +| `only_main_content` | boolean | Strip nav, footer, and boilerplate. | +| `timeout` | integer | Timeout in milliseconds. Default: 60000, min: 1000, max: 300000. | +| `wait_for` | integer | Wait for page render (milliseconds). | +| `mobile` | boolean | Use a mobile viewport. | +| `parsers` | list | Parser configs (e.g. PDF parsing). | +| `actions` | list | Pre-scrape browser actions. | +| `location` | keyword list | `[country: "US", languages: ["en-US"]]` for geo-aware scraping. | +| `skip_tls_verification` | boolean | Skip TLS verification. | +| `remove_base64_images` | boolean | Drop base64 images from markdown. | +| `block_ads` | boolean | Block ads and cookie popups. | +| `proxy` | atom | Proxy: `:basic`, `:enhanced`, `:auto`. | +| `max_age` | integer | Use cached data up to this age (milliseconds). Default: 2 days. | +| `min_age` | integer | Use cached data only if at least this old (milliseconds). | +| `store_in_cache` | boolean | Cache the result. | +| `lockdown` | boolean | Only serve cached results. | +| `profile` | keyword list | `[name: "my-profile", save_changes: true]` for persistent browser profiles. | + +## Interact + +### Why use it + +Use interact for code-based control of the browser session tied to a scrape job. The Elixir SDK requires `code` (no `prompt` parameter). + +### Preferred SDK method + +`Firecrawl.interact_with_scrape_browser_session(job_id, params \\ [], opts \\ [])` + +### Example + +```elixir +{:ok, scrape_res} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com", + formats: ["markdown"] +) + +job_id = get_in(scrape_res.body, ["data", "metadata", "scrapeId"]) + +{:ok, result} = Firecrawl.interact_with_scrape_browser_session( + job_id, + code: "console.log(await page.title());", + language: :node, + timeout: 60 +) + +# Stop the session when done +{:ok, _} = Firecrawl.stop_interactive_scrape_browser_session(job_id) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `job_id` | string | Scrape job ID from scrape response metadata `scrapeId`. | +| `code` | string | Code to run in the browser session. Required. | +| `language` | atom | Runtime: `:python`, `:node`, `:bash`. | +| `timeout` | integer | Execution timeout in seconds. | + +### Stop session + +`Firecrawl.stop_interactive_scrape_browser_session(job_id, opts \\ [])` + +## Notes + +- The Elixir client is **auto-generated from the OpenAPI spec** — function names are derived from operation IDs and may look verbose (e.g. `scrape_and_extract_from_url` instead of `scrape`). +- All functions return `{:ok, %Req.Response{}}` or `{:error, exception}`. Bang variants (e.g. `scrape_and_extract_from_url!`) raise on error. +- Parameters use `snake_case` keyword lists, but are converted to `camelCase` JSON for the API. +- Atoms in parameter values (e.g. `:node`, `:web`) are converted to strings in the request body. +- Per-call options (`api_key:`, `base_url:`) are passed in the second argument (opts). + +## Source Of Truth + +- `firecrawl/apps/elixir-sdk/lib/firecrawl.ex` +- `firecrawl/apps/elixir-sdk/mix.exs` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/java.mdx b/agent-quickstart/java.mdx new file mode 100644 index 000000000..4c54259bb --- /dev/null +++ b/agent-quickstart/java.mdx @@ -0,0 +1,227 @@ +--- +title: "Java Agent Quickstart" +description: "Canonical Firecrawl Java quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Java Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl-java`) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +Maven: + +```xml + + com.firecrawl + firecrawl-java + 1.12.1 + +``` + +Gradle: + +```gradle +implementation("com.firecrawl:firecrawl-java:1.12.1") +``` + +## Authenticate + +```java +import com.firecrawl.client.FirecrawlClient; + +FirecrawlClient client = FirecrawlClient.builder() + .apiKey(System.getenv("FIRECRAWL_API_KEY")) + .build(); + +// Or read from FIRECRAWL_API_KEY env var or firecrawl.apiKey system property: +// FirecrawlClient client = FirecrawlClient.fromEnv(); +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +- `client.search(query)` → `SearchData` +- `client.search(query, options)` → `SearchData` + +### Example + +```java +import com.firecrawl.models.SearchData; +import com.firecrawl.models.SearchOptions; +import com.firecrawl.models.ScrapeOptions; + +SearchOptions options = SearchOptions.builder() + .sources(List.of("web")) + .limit(5) + .scrapeOptions( + ScrapeOptions.builder() + .formats(List.of("markdown")) + .onlyMainContent(true) + .build() + ) + .build(); + +SearchData results = client.search("site:docs.firecrawl.dev webhook retries", options); +List> web = results.getWeb(); +``` + +### Return value + +`SearchData` has `getWeb()`, `getNews()`, `getImages()` — each returns `List>` (may be null). Do not treat `SearchData` as a directly iterable list. + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | String | Search query. Use `site:example.com` to limit to a domain. | +| `options.sources` | List\ | Sources to search: `"web"`, `"news"`, `"images"`. | +| `options.categories` | List\ | Filter by category: `"github"`, `"research"`, `"pdf"`. | +| `options.limit` | Integer | Cap the number of results. | +| `options.tbs` | String | Time-based filter (e.g. `"qdr:d"`, `"qdr:w"`). | +| `options.location` | String | Localized results. | +| `options.ignoreInvalidURLs` | Boolean | Drop URLs that cannot be scraped. | +| `options.timeout` | Integer | Request timeout in milliseconds. | +| `options.scrapeOptions` | ScrapeOptions | Scrape each search result. See Scrape parameters. | +| `options.includeDomains` | List\ | Only include results from these domains. | +| `options.excludeDomains` | List\ | Exclude results from these domains. | +| `options.highlights` | Boolean | Return highlights (defaults to true). | +| `options.integration` | String | Integration identifier. | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +- `client.scrape(url)` → `Document` +- `client.scrape(url, options)` → `Document` + +### Example + +```java +import com.firecrawl.models.Document; +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.JsonFormat; + +ScrapeOptions options = ScrapeOptions.builder() + .formats(List.of( + "markdown", + "links", + JsonFormat.builder().prompt("Extract plan names and prices.").build() + )) + .onlyMainContent(true) + .waitFor(1000) + .build(); + +Document doc = client.scrape("https://example.com/pricing", options); +System.out.println(doc.getMarkdown()); +System.out.println(doc.getJson()); +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `url` | String | The page URL to scrape. | +| `options.formats` | List\ | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Objects: `JsonFormat.builder().prompt(...).schema(...).build()`, `QuestionFormat`, `HighlightsFormat`. | +| `options.headers` | Map\ | Custom request headers. | +| `options.includeTags` | List\ | Include only specific HTML tags. | +| `options.excludeTags` | List\ | Exclude specific HTML tags. | +| `options.onlyMainContent` | Boolean | Strip nav, footer, and boilerplate. | +| `options.timeout` | Integer | Timeout in milliseconds. | +| `options.waitFor` | Integer | Wait for page render (milliseconds). | +| `options.mobile` | Boolean | Use a mobile viewport. | +| `options.parsers` | List\ | Parser configs (e.g. `"pdf"` or `Map.of("type", "pdf", "maxPages", 5)`). | +| `options.actions` | List\\> | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `scrape`, `screenshot`, `executeJavascript`, `pdf`. | +| `options.location` | LocationConfig | `LocationConfig.builder().country("US").languages(List.of("en-US")).build()`. | +| `options.skipTlsVerification` | Boolean | Skip TLS verification. | +| `options.removeBase64Images` | Boolean | Drop base64 images from markdown. | +| `options.blockAds` | Boolean | Block ads and cookie popups. | +| `options.proxy` | String | Proxy: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or custom URL. | +| `options.maxAge` | Long | Use cached data up to this age (milliseconds). | +| `options.storeInCache` | Boolean | Cache the result. | +| `options.lockdown` | Boolean | Only serve cached results. | +| `options.integration` | String | Integration identifier. | + +## Interact + +### Why use it + +Use interact when a page requires browser actions or code execution after a scrape starts. The Java SDK exposes code-based interactions only (no `prompt` parameter). + +### Preferred SDK method + +- `client.interact(jobId, code)` → `BrowserExecuteResponse` +- `client.interact(jobId, code, language, timeout)` → `BrowserExecuteResponse` + +### Example + +```java +import com.firecrawl.models.BrowserExecuteResponse; +import com.firecrawl.models.Document; +import com.firecrawl.models.ScrapeOptions; + +Document doc = client.scrape("https://example.com", + ScrapeOptions.builder().formats(List.of("markdown")).build()); +String jobId = (String) doc.getMetadata().get("scrapeId"); + +BrowserExecuteResponse result = client.interact( + jobId, + "console.log(await page.title());", + "node", + 60 +); + +System.out.println(result.getStdout()); + +// Stop the session when done +client.stopInteractiveBrowser(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `jobId` | String | Scrape job ID from `document.getMetadata().get("scrapeId")`. | +| `code` | String | Code to run in the browser session. | +| `language` | String | Runtime: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `timeout` | Integer | Execution timeout in seconds (1-300). Null uses API default (30s). | +| `origin` | String | Optional origin label (fifth parameter overload). | + +### Stop session + +`client.stopInteractiveBrowser(jobId)` → `BrowserDeleteResponse` + +### Response fields + +`BrowserExecuteResponse`: `isSuccess()`, `getStdout()`, `getStderr()`, `getResult()`, `getExitCode()`, `getKilled()`, `getError()`. + +`BrowserDeleteResponse`: `isSuccess()`, `getSessionDurationMs()`, `getCreditsBilled()`, `getError()`. + +## Notes + +- Deprecated aliases: `scrapeExecute` → `interact`; `deleteScrapeBrowser` → `stopInteractiveBrowser`. +- The Java SDK does not have a `prompt` parameter on `interact` (code-based only, unlike some other SDKs). +- All options classes use the builder pattern: `ScrapeOptions.builder()...build()`. +- Every synchronous method has a corresponding `*Async(...)` variant returning `CompletableFuture`. +- Java 11+ is required. + +## Source Of Truth + +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/` (all model classes) +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/node.mdx b/agent-quickstart/node.mdx new file mode 100644 index 000000000..a028bff41 --- /dev/null +++ b/agent-quickstart/node.mdx @@ -0,0 +1,197 @@ +--- +title: "Node.js Agent Quickstart" +description: "Canonical Firecrawl Node.js quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Node.js Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` JS SDK) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +```bash +npm install firecrawl +``` + +## Authenticate + +```ts +import { Firecrawl } from "firecrawl"; + +const client = new Firecrawl({ + apiKey: process.env.FIRECRAWL_API_KEY, + // apiUrl: "https://api.firecrawl.dev" // optional; falls back to FIRECRAWL_API_URL or cloud default +}); +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. For multi-step interactive flows, prefer `interact` over scrape-time `actions`. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, options?)` → `Promise` + +### Example + +```ts +const results = await client.search("site:docs.firecrawl.dev webhook retries", { + sources: ["web"], + limit: 5, + scrapeOptions: { + formats: ["markdown"], + onlyMainContent: true, + }, +}); + +for (const item of results.web ?? []) { + console.log(item.url, item.title); +} +``` + +### Return value + +`SearchData` has optional arrays — `web`, `news`, `images`, `developer`. Each entry is either a lightweight result or a full `Document` when `scrapeOptions` hydrated the hit. + +**Wrong turn to avoid:** `search()` does not return `{ data: [...] }`. Web results are in `result.web`, not `result.data`. + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | string | Search query. Use `site:example.com` to limit to a domain. | +| `options.sources` | array | Sources to search: `"web"`, `"news"`, `"images"`. | +| `options.categories` | array | Filter by category: `"github"`, `"research"`, `"pdf"`, `"developer"`. | +| `options.limit` | number | Cap the number of results. | +| `options.tbs` | string | Time-based filter (e.g. `"qdr:d"`, `"qdr:w"`). | +| `options.location` | string | Localized results (e.g. `"San Francisco,California,United States"`). | +| `options.ignoreInvalidURLs` | boolean | Drop URLs that cannot be scraped. | +| `options.timeout` | number | Request timeout in milliseconds. | +| `options.scrapeOptions` | ScrapeOptions | Scrape each search result. See Scrape parameters. | +| `options.includeDomains` | string[] | Only include results from these domains. | +| `options.excludeDomains` | string[] | Exclude results from these domains. | +| `options.highlights` | boolean | Return highlights (defaults to true). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, options?)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com/pricing", { + formats: [ + "markdown", + "links", + { type: "json", prompt: "Extract plan names and prices." }, + ], + onlyMainContent: true, + waitFor: 1000, +}); + +console.log(doc.markdown); +console.log(doc.json); +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `url` | string | The page URL to scrape. | +| `options.formats` | array | Output formats. Plain strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Object forms: `{ type: "json", prompt?, schema? }`, `{ type: "screenshot", fullPage?, quality?, viewport? }`, `{ type: "changeTracking", modes: ["git-diff"\|"json"], schema?, prompt?, tag? }`, `{ type: "attributes", selectors: [{ selector, attribute }] }`, `{ type: "question", question }`, `{ type: "highlights", query }`. Note: plain string `"json"` is rejected by the SDK — use the object form. | +| `options.headers` | Record | Custom request headers. | +| `options.includeTags` | string[] | Include only specific HTML tags. | +| `options.excludeTags` | string[] | Exclude specific HTML tags. | +| `options.onlyMainContent` | boolean | Strip nav, footer, and boilerplate. | +| `options.timeout` | number | Timeout in milliseconds. | +| `options.waitFor` | number | Wait for page render (milliseconds). | +| `options.mobile` | boolean | Use a mobile viewport. | +| `options.parsers` | array | Parser configs, e.g. `"pdf"` or `{ type: "pdf", mode?: "fast"\|"auto"\|"ocr", maxPages?: number }`. | +| `options.actions` | array | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `scrape`, `screenshot`, `executeJavascript`, `pdf`. | +| `options.location` | object | `{ country, languages }` for geo-aware scraping. | +| `options.skipTlsVerification` | boolean | Skip TLS verification. | +| `options.removeBase64Images` | boolean | Drop base64 images from markdown. | +| `options.fastMode` | boolean | Faster scrapes with reduced fidelity. | +| `options.blockAds` | boolean | Block ads and cookie popups. | +| `options.proxy` | string | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or a custom URL. | +| `options.maxAge` | number | Use cached data up to this age (milliseconds). | +| `options.minAge` | number | Use cached data only if at least this old (milliseconds). | +| `options.storeInCache` | boolean | Cache the result. | +| `options.profile` | object | `{ name, saveChanges? }` for persistent browser profiles. | + +## Interact + +### Why use it + +Use `interact` for code or natural-language control of the browser session tied to a scrape job (via `metadata.scrapeId`). The SDK requires at least one of `code` or `prompt`. + +### Preferred SDK method + +`client.interact(jobId, args)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com", { formats: ["markdown"] }); +const jobId = doc.metadata?.scrapeId; +if (!jobId) throw new Error("Missing scrapeId from scrape response"); + +// Natural-language interaction +const result = await client.interact(jobId, { + prompt: "Click the pricing tab and summarize the plans.", +}); + +// Code-based interaction +const codeResult = await client.interact(jobId, { + code: "console.log(await page.title());", + language: "node", + timeout: 60, +}); + +// Stop the session when done +await client.stopInteraction(jobId); +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `jobId` | string | Scrape job ID from `document.metadata.scrapeId`. | +| `args.code` | string | Code to run in the browser session (e.g. Playwright `page` usage). | +| `args.prompt` | string | Natural-language instruction for the browser agent. | +| `args.language` | string | Runtime: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `args.timeout` | number | Execution timeout in seconds. | + +At least one of `code` or `prompt` must be non-empty. + +### Stop session + +`client.stopInteraction(jobId)` → `Promise` + +## Notes + +- Deprecated aliases: `scrapeExecute` → `interact`; `stopInteractiveBrowser` / `deleteScrapeBrowser` → `stopInteraction`. +- The default `Firecrawl` export is the v2 client; v1 remains under `client.v1`. +- Zod schemas passed to `formats` (for `json` or `changeTracking`) are converted to JSON Schema by the SDK. +- The package declares **Node.js >= 22** in `engines`. + +## Source Of Truth + +- `firecrawl/apps/js-sdk/firecrawl/src/index.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/python.mdx b/agent-quickstart/python.mdx new file mode 100644 index 000000000..bbcc5467c --- /dev/null +++ b/agent-quickstart/python.mdx @@ -0,0 +1,204 @@ +--- +title: "Python Agent Quickstart" +description: "Canonical Firecrawl Python quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Python Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl-py`) and the v2 OpenAPI spec. Method names and parameters match the v2 client. + +## Install + +```bash +pip install firecrawl-py +``` + +## Authenticate + +```python +import os +from firecrawl import Firecrawl + +client = Firecrawl(api_key=os.environ.get("FIRECRAWL_API_KEY")) +# client = Firecrawl(api_key="fc-...", api_url="https://api.firecrawl.dev") +``` + +An async client is also available: + +```python +from firecrawl import AsyncFirecrawl + +client = AsyncFirecrawl(api_key=os.environ.get("FIRECRAWL_API_KEY")) +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, **options)` → `SearchData` + +### Example + +```python +results = client.search( + "site:docs.firecrawl.dev webhook retries", + sources=["web"], + limit=5, + scrape_options={ + "formats": ["markdown"], + "only_main_content": True, + }, +) + +for item in results.web or []: + print(getattr(item, "url", None), getattr(item, "title", None)) +``` + +### Return value + +Returns a `SearchData` model with optional lists: `web`, `news`, `images`, `developer`. Each entry is either a lightweight result or a full `Document` when `scrape_options` hydrated the hit. + +**Wrong turn to avoid:** `search()` does not return `{ data: [...] }`. Web results are in `result.web`, not `result.data`. + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | str | Search query. Use `site:example.com` to limit to a domain. | +| `sources` | list | Sources to search: `"web"`, `"news"`, `"images"`. | +| `categories` | list | Filter by category: `"github"`, `"research"`, `"pdf"`, `"developer"`. | +| `limit` | int | Cap the number of results. Default: `5`. | +| `tbs` | str | Time-based filter (e.g. `"qdr:d"`, `"qdr:w"`). | +| `location` | str | Localized results (e.g. `"San Francisco,California,United States"`). | +| `ignore_invalid_urls` | bool | Drop URLs that cannot be scraped. | +| `timeout` | int | Request timeout in milliseconds. Default: `300000`. | +| `scrape_options` | dict or ScrapeOptions | Scrape each search result. See Scrape parameters. | +| `include_domains` | list[str] | Only include results from these domains. | +| `exclude_domains` | list[str] | Exclude results from these domains. | +| `highlights` | bool | Return highlights (defaults to true). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, **options)` → `Document` + +### Example + +```python +doc = client.scrape( + "https://example.com/pricing", + formats=[ + "markdown", + "links", + {"type": "json", "prompt": "Extract plan names and prices."}, + ], + only_main_content=True, + wait_for=1000, +) + +print(doc.markdown) +print(doc.json) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `url` | str | The page URL to scrape. | +| `formats` | list | Output formats. Plain strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"json"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Object forms: `{"type": "json", "prompt": ..., "schema": ...}`, `{"type": "screenshot", "fullPage": ..., "quality": ..., "viewport": ...}`, `{"type": "changeTracking", "modes": [...], "schema": ..., "prompt": ..., "tag": ...}`, `{"type": "attributes", "selectors": [{"selector": ..., "attribute": ...}]}`, `{"type": "question", "question": ...}`, `{"type": "highlights", "query": ...}`. | +| `headers` | dict | Custom request headers. | +| `include_tags` | list[str] | Include only specific HTML tags. | +| `exclude_tags` | list[str] | Exclude specific HTML tags. | +| `only_main_content` | bool | Strip nav, footer, and boilerplate. | +| `timeout` | int | Timeout in milliseconds. | +| `wait_for` | int | Wait for page render (milliseconds). | +| `mobile` | bool | Use a mobile viewport. | +| `parsers` | list | Parser configs, e.g. `"pdf"` or `{"type": "pdf", "mode": "fast"|"auto"|"ocr", "maxPages": int}`. | +| `actions` | list | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `scrape`, `screenshot`, `executeJavascript`, `pdf`. | +| `location` | dict | `{"country": ..., "languages": [...]}` for geo-aware scraping. | +| `skip_tls_verification` | bool | Skip TLS verification. | +| `remove_base64_images` | bool | Drop base64 images from markdown. | +| `fast_mode` | bool | Faster scrapes with reduced fidelity. | +| `block_ads` | bool | Block ads and cookie popups. | +| `proxy` | str | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. | +| `max_age` | int | Use cached data up to this age (milliseconds). | +| `min_age` | int | Use cached data only if at least this old (milliseconds). | +| `store_in_cache` | bool | Cache the result. | +| `lockdown` | bool | Only serve cached results. | +| `profile` | dict | `{"name": ..., "save_changes": ...}` for persistent browser profiles. | + +## Interact + +### Why use it + +Use `interact` for code or natural-language control of the browser session tied to a scrape job (via `metadata.scrape_id`). At least one of `code` or `prompt` is required. + +### Preferred SDK method + +`client.interact(job_id, code, *, prompt, language, timeout)` → response + +### Example + +```python +doc = client.scrape("https://example.com", formats=["markdown"]) +job_id = doc.metadata.scrape_id +if not job_id: + raise ValueError("Missing scrape_id from scrape response") + +# Natural-language interaction +result = client.interact(job_id, code="", prompt="Click the pricing tab and summarize the plans.") + +# Code-based interaction +code_result = client.interact( + job_id, + code="console.log(await page.title());", + language="node", + timeout=60, +) + +# Stop the session when done +client.stop_interaction(job_id) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `job_id` | str | Scrape job ID from `document.metadata.scrape_id`. | +| `code` | str | Code to run in the browser session (positional argument). | +| `prompt` | str | Natural-language instruction for the browser agent. | +| `language` | str | Runtime: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `timeout` | int | Execution timeout in seconds. | + +### Stop session + +`client.stop_interaction(job_id)` + +## Notes + +- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser` / `delete_scrape_browser` → `stop_interaction`. +- `FirecrawlApp` is a backward-compat alias for `Firecrawl`. +- The import is `from firecrawl import Firecrawl` (package name is `firecrawl-py` on PyPI, `firecrawl` for import). +- Parameter names use `snake_case` in the Python SDK. + +## Source Of Truth + +- `firecrawl/apps/python-sdk/firecrawl/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/types.py` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/rust.mdx b/agent-quickstart/rust.mdx new file mode 100644 index 000000000..4aed94cdf --- /dev/null +++ b/agent-quickstart/rust.mdx @@ -0,0 +1,213 @@ +--- +title: "Rust Agent Quickstart" +description: "Canonical Firecrawl Rust quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Rust Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` crate) and the v2 OpenAPI spec. Method names and parameters match the SDK public API. + +## Install + +```bash +cargo add firecrawl +``` + +Crate: **`firecrawl`** on crates.io. + +## Authenticate + +```rust +use firecrawl::Client; + +let client = Client::new("fc-your-api-key"); + +// Self-hosted: +// let client = Client::new_selfhosted("http://localhost:3002", Some("fc-your-api-key")); +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, SearchOptions, SearchSource, ScrapeOptions, Format}; + +let options = SearchOptions { + sources: Some(vec![SearchSource::Web]), + limit: Some(5), + scrape_options: Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + only_main_content: Some(true), + ..Default::default() + }), + ..Default::default() +}; + +let results = client + .search("site:docs.firecrawl.dev webhook retries", Some(options)) + .await?; +``` + +### Return value + +`SearchResponse` contains `SearchData` with optional arrays: `web`, `news`, `images`. Each web entry is either a `SearchResultWeb` or a full `Document` (when `scrape_options` hydrated the result). + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | &str | Search query. Use `site:example.com` to limit to a domain. | +| `options.sources` | `Option>` | Sources: `Web`, `News`, `Images`. | +| `options.categories` | `Option>` | Filter by category: `Github`, `Research`, `Pdf`. | +| `options.limit` | `Option` | Cap the number of results. Default: 5, max: 20. | +| `options.tbs` | `Option` | Time-based filter (e.g. `"qdr:d"`, `"qdr:w"`). | +| `options.location` | `Option` | Localized results. | +| `options.ignore_invalid_urls` | `Option` | Drop URLs that cannot be scraped. | +| `options.timeout` | `Option` | Request timeout in milliseconds. | +| `options.scrape_options` | `Option` | Scrape each search result. See Scrape parameters. | +| `options.include_domains` | `Option>` | Only include results from these domains. | +| `options.exclude_domains` | `Option>` | Exclude results from these domains. | +| `options.highlights` | `Option` | Return highlights (defaults to true). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format, JsonOptions}; + +let options = ScrapeOptions { + formats: Some(vec![Format::Markdown, Format::Links, Format::Json]), + json_options: Some(JsonOptions { + prompt: Some("Extract plan names and prices.".to_string()), + ..Default::default() + }), + only_main_content: Some(true), + wait_for: Some(1000), + ..Default::default() +}; + +let doc = client.scrape("https://example.com/pricing", Some(options)).await?; +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `url` | &str | The page URL to scrape. | +| `options.formats` | `Option>` | Output formats: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Product`, `Menu`, `Audio`, `Video`, `Question`, `Highlights`, `Query`. | +| `options.json_options` | `Option` | JSON extraction options: `prompt`, `schema`. | +| `options.screenshot_options` | `Option` | Screenshot options: `full_page`, `quality`, `viewport`. | +| `options.change_tracking_options` | `Option` | Change tracking options: `modes`, `schema`, `prompt`, `tag`. | +| `options.attribute_selectors` | `Option>` | Attribute extraction selectors. | +| `options.headers` | `Option>` | Custom request headers. | +| `options.include_tags` | `Option>` | Include only specific HTML tags. | +| `options.exclude_tags` | `Option>` | Exclude specific HTML tags. | +| `options.only_main_content` | `Option` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `Option` | Timeout in milliseconds. | +| `options.wait_for` | `Option` | Wait for page render (milliseconds). | +| `options.mobile` | `Option` | Use a mobile viewport. | +| `options.parsers` | `Option>` | Parser configs (e.g. PDF parsing). | +| `options.actions` | `Option>` | Pre-scrape browser actions. | +| `options.location` | `Option` | `country` and `languages` for geo-aware scraping. | +| `options.skip_tls_verification` | `Option` | Skip TLS verification. | +| `options.remove_base64_images` | `Option` | Drop base64 images from markdown. | +| `options.fast_mode` | `Option` | Faster scrapes with reduced fidelity. | +| `options.block_ads` | `Option` | Block ads and cookie popups. | +| `options.proxy` | `Option` | Proxy: `Basic`, `Stealth`, `Enhanced`, `Auto`. | +| `options.max_age` | `Option` | Use cached data up to this age (seconds). | +| `options.min_age` | `Option` | Use cached data only if at least this old (seconds). | +| `options.store_in_cache` | `Option` | Cache the result. | +| `options.lockdown` | `Option` | Only serve cached results. | +| `options.profile` | `Option` | Persistent browser profile: `name`, `save_changes`. | + +## Interact + +### Why use it + +Use `interact` for code-based control of the browser session tied to a scrape job. At least one of `code` or `prompt` is required. + +### Preferred SDK method + +`client.interact(job_id, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, ScrapeExecuteOptions, ScrapeExecuteLanguage, Format}; + +let doc = client + .scrape("https://example.com", Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + ..Default::default() + })) + .await?; + +let job_id = doc.metadata + .and_then(|m| m.get("scrapeId").and_then(|v| v.as_str().map(String::from))) + .expect("Missing scrapeId"); + +let result = client + .interact(&job_id, ScrapeExecuteOptions { + code: Some("console.log(await page.title());".to_string()), + language: Some(ScrapeExecuteLanguage::Node), + timeout: Some(60), + ..Default::default() + }) + .await?; + +// Stop the session when done +client.stop_interaction(&job_id).await?; +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `job_id` | &str | Scrape job ID from document metadata `scrapeId`. | +| `options.code` | `Option` | Code to run in the browser session. | +| `options.prompt` | `Option` | Natural-language instruction for the browser agent. | +| `options.language` | `Option` | Runtime: `Python`, `Node`, `Bash`. Default: `Node`. | +| `options.timeout` | `Option` | Execution timeout in seconds. | + +### Stop session + +`client.stop_interaction(job_id)` → `Result` + +## Notes + +- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser` / `delete_scrape_browser` → `stop_interaction`. +- All `ScrapeOptions` fields are `Option` and derive `Default`, so use `..Default::default()` for partial construction. +- The SDK auto-sets `origin` to `"rust-sdk@{version}"` on every request. +- Crate uses async by default (requires a `tokio` or equivalent runtime). + +## Source Of Truth + +- `firecrawl/apps/rust-sdk/src/client.rs` +- `firecrawl/apps/rust-sdk/src/scrape.rs` +- `firecrawl/apps/rust-sdk/src/search.rs` +- `firecrawl/apps/rust-sdk/src/types.rs` +- `firecrawl-docs/api-reference/v2-openapi.json`