diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4b7392f998..73e527ca23 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "description": "Scrape, search, crawl, and map the web with a single command.", "skills": [ "./skills/firecrawl-agent", - "./skills/firecrawl-cli", + "./skills/firecrawl", "./skills/firecrawl-crawl", "./skills/firecrawl-download", "./skills/firecrawl-interact", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e59ca3ab1c..499f8cd18b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -7,7 +7,7 @@ }, "skills": [ "./skills/firecrawl-agent", - "./skills/firecrawl-cli", + "./skills/firecrawl", "./skills/firecrawl-crawl", "./skills/firecrawl-download", "./skills/firecrawl-interact", diff --git a/README.md b/README.md index 6576734427..68045f23a2 100644 --- a/README.md +++ b/README.md @@ -667,10 +667,10 @@ firecrawl agent "Find the top 5 competitors of Notion and their pricing" --wait firecrawl agent "Get all blog post titles and dates" --urls https://blog.example.com --max-credits 100 --wait # Use higher accuracy model for complex extraction -firecrawl agent "Extract detailed technical specifications" --model spark-1-pro --wait --pretty +firecrawl agent "Extract detailed technical specifications" --model spark-1-pro --wait --json --pretty # Save structured results to file -firecrawl agent "Extract contact information" --schema-file ./contact-schema.json --wait -o contacts.json --pretty +firecrawl agent "Extract contact information" --schema-file ./contact-schema.json --wait --json -o contacts.json --pretty # Check job status without waiting firecrawl agent abc123-def456-... --json diff --git a/package.json b/package.json index 097a4ee0e5..0aec596392 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,8 @@ "clean": "rm -rf dist", "prepublishOnly": "pnpm run build", "prepare": "husky", - "format": "prettier --write \"src/**/*.{ts,json}\" \"*.{json,md}\"", - "format:check": "prettier --check \"src/**/*.{ts,json}\" \"*.{json,md}\"", + "format": "prettier --write \"src/**/*.{ts,json}\" \"skills/**/*.md\" \"*.{json,md}\"", + "format:check": "prettier --check \"src/**/*.{ts,json}\" \"skills/**/*.md\" \"*.{json,md}\"", "type-check": "tsc --noEmit", "test:watch": "vitest", "test": "vitest run", diff --git a/skills/firecrawl-agent/SKILL.md b/skills/firecrawl-agent/SKILL.md index 18fd81bff5..5258fa8730 100644 --- a/skills/firecrawl-agent/SKILL.md +++ b/skills/firecrawl-agent/SKILL.md @@ -4,7 +4,7 @@ description: | AI-powered autonomous data extraction that navigates complex sites and returns structured JSON. Use this skill when the user wants structured data from websites, needs to extract pricing tiers, product listings, directory entries, or any data as JSON with a schema. Triggers on "extract structured data", "get all the products", "pull pricing info", "extract as JSON", or when the user provides a JSON schema for website data. More powerful than simple scraping for multi-page structured extraction. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl agent @@ -21,31 +21,51 @@ AI-powered autonomous extraction. The agent navigates sites and extracts structu ```bash # Extract structured data -firecrawl agent "extract all pricing tiers" --wait -o .firecrawl/pricing.json +firecrawl agent "extract all pricing tiers" --wait --json -o .firecrawl/pricing.json # With a JSON schema for structured output -firecrawl agent "extract products" --schema '{"type":"object","properties":{"name":{"type":"string"},"price":{"type":"number"}}}' --wait -o .firecrawl/products.json +firecrawl agent "extract products" --schema '{"type":"object","properties":{"name":{"type":"string"},"price":{"type":"number"}}}' --wait --json -o .firecrawl/products.json # Focus on specific pages -firecrawl agent "get feature list" --urls "" --wait -o .firecrawl/features.json +firecrawl agent "get feature list" --urls "" --wait --json -o .firecrawl/features.json +``` + +## Job IDs + +Without `--wait`, the command returns a job ID. A UUID positional argument is auto-detected as a status check: + +```bash +# Check once (equivalent to adding --status) +firecrawl agent "" + +# Wait on an existing job, polling every 10 seconds for up to 5 minutes +firecrawl agent "" --wait --poll-interval 10 --timeout 300 + +# Cancel an active job +firecrawl agent "" --cancel ``` ## Options -| Option | Description | -| ---------------------- | ----------------------------------------- | -| `--urls ` | Starting URLs for the agent | -| `--model ` | Model to use: spark-1-mini or spark-1-pro | -| `--schema ` | JSON schema for structured output | -| `--schema-file ` | Path to JSON schema file | -| `--max-credits ` | Credit limit for this agent run | -| `--wait` | Wait for agent to complete | -| `--pretty` | Pretty print JSON output | -| `-o, --output ` | Output file path | +| Option | Description | +| --------------------------- | ----------------------------------------------------- | +| `--urls ` | Starting URLs for the agent | +| `--model ` | Model to use: spark-1-mini or spark-1-pro | +| `--schema ` | JSON schema for structured output | +| `--schema-file ` | Path to JSON schema file | +| `--max-credits ` | Credit limit for this agent run | +| `--status` | Check a job ID's status | +| `--cancel` | Cancel an active job ID | +| `--wait` | Wait for agent to complete | +| `--poll-interval ` | Polling interval while waiting (default: 5 seconds) | +| `--timeout ` | Stop waiting after this duration (default: none) | +| `--json` | Output as JSON | +| `--pretty` | Pretty print JSON (`--wait` results require `--json`) | +| `-o, --output ` | Output file path | ## Tips -- Always use `--wait` to get results inline. Without it, returns a job ID. +- Use `--wait` for inline results; without it you get a job ID (see [Job IDs](#job-ids)). - Use `--schema` for predictable, structured output — otherwise the agent returns freeform data. - Agent runs consume more credits than simple scrapes. Use `--max-credits` to cap spending. - For simple single-page extraction, prefer `scrape` — it's faster and cheaper. diff --git a/skills/firecrawl-crawl/SKILL.md b/skills/firecrawl-crawl/SKILL.md index ca6e6b5aaf..1d99f48a54 100644 --- a/skills/firecrawl-crawl/SKILL.md +++ b/skills/firecrawl-crawl/SKILL.md @@ -4,18 +4,20 @@ description: | Bulk extract content from an entire website or site section. Use this skill when the user wants to crawl a site, extract all pages from a docs section, bulk-scrape multiple pages following links, or says "crawl", "get all the pages", "extract everything under /docs", "bulk extract", or needs content from many pages on the same site. Handles depth limits, path filtering, and concurrent extraction. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl crawl Bulk extract content from a website. Crawls pages following links up to a depth/limit. +**Prerequisite:** `crawl` requires authentication (no keyless free tier); without credentials the CLI prompts an interactive login. + ## When to use - You need content from many pages on a site (e.g., all `/docs/`) - You want to extract an entire site section -- Step 4 in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → **crawl** → interact +- Step 4 in the [workflow escalation pattern](../firecrawl/SKILL.md): search → scrape → map + scrape → **crawl** → monitor → interact ## Quick start @@ -47,9 +49,9 @@ firecrawl crawl ## Tips -- Always use `--wait` when you need the results immediately. Without it, crawl returns a job ID for async polling. +- Always use `--wait` when you need the results immediately. It has no default timeout; use `--timeout ` to bound polling. Without `--wait`, crawl returns a job ID for async polling. - Use `--include-paths` to scope the crawl — don't crawl an entire site when you only need one section. -- Crawl consumes credits per page. Check `firecrawl credit-usage` before large crawls. +- Crawl consumes credits per page. Check `firecrawl credit-usage` before large crawls (`credit-usage` requires authentication). ## See also diff --git a/skills/firecrawl-download/SKILL.md b/skills/firecrawl-download/SKILL.md index d2beeb7c66..ece43a10d5 100644 --- a/skills/firecrawl-download/SKILL.md +++ b/skills/firecrawl-download/SKILL.md @@ -4,14 +4,16 @@ description: | Download an entire website as local files — markdown, screenshots, or multiple formats per page. Use this skill when the user wants to save a site locally, download documentation for offline use, bulk-save pages as files, or says "download the site", "save as local files", "offline copy", "download all the docs", or "save for reference". Combines site mapping and scraping into organized local directories. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- -# firecrawl download +# firecrawl download (invoked as `firecrawl x download`) -> **Experimental.** Convenience command that combines `map` + `scrape` to save an entire site as local files. +> **Experimental.** `download` is available under the `firecrawl x` command group. -Maps the site first to discover pages, then scrapes each one into nested directories under `.firecrawl/`. All scrape options work with download. Always pass `-y` to skip the confirmation prompt. +**Prerequisite:** `download` requires authentication (no keyless free tier); without credentials the CLI prompts an interactive login. + +Maps the site origin first to discover pages, then scrapes each one into nested directories under `.firecrawl/`. Use `--include-paths` to scope a non-root URL to one section. Supported scrape options are listed below. Always pass `-y` to skip the confirmation prompt. ## When to use @@ -22,24 +24,24 @@ Maps the site first to discover pages, then scrapes each one into nested directo ## Quick start ```bash -# Interactive wizard (picks format, screenshots, paths for you) -firecrawl download https://docs.example.com +# Interactive wizard (humans at a TTY only — agents must pass -y or the command blocks on a prompt) +firecrawl x download https://docs.example.com # With screenshots -firecrawl download https://docs.example.com --screenshot --limit 20 -y +firecrawl x download https://docs.example.com --screenshot --limit 20 -y # Multiple formats (each saved as its own file per page) -firecrawl download https://docs.example.com --format markdown,links --screenshot --limit 20 -y +firecrawl x download https://docs.example.com --format markdown,links --screenshot --limit 20 -y # Creates per page: index.md + links.txt + screenshot.png # Filter to specific sections -firecrawl download https://docs.example.com --include-paths "/features,/sdks" +firecrawl x download https://docs.example.com --include-paths "/features,/sdks" -y # Skip translations -firecrawl download https://docs.example.com --exclude-paths "/zh,/ja,/fr,/es,/pt-BR" +firecrawl x download https://docs.example.com --exclude-paths "/zh,/ja,/fr,/es,/pt-BR" -y # Full combo -firecrawl download https://docs.example.com \ +firecrawl x download https://docs.example.com \ --include-paths "/features,/sdks" \ --exclude-paths "/zh,/ja" \ --only-main-content \ @@ -58,9 +60,11 @@ firecrawl download https://docs.example.com \ | `--allow-subdomains` | Include subdomain pages | | `-y` | Skip confirmation prompt (always use in automated flows) | -## Scrape options (all work with download) +## Supported scrape options + +Only the options listed below are supported: -`-f `, `-H`, `-S`, `--screenshot`, `--full-page-screenshot`, `--only-main-content`, `--include-tags`, `--exclude-tags`, `--wait-for`, `--max-age`, `--country`, `--languages` +`-f `, `-H`, `-S`, `--lockdown`, `--screenshot`, `--full-page-screenshot`, `--only-main-content`, `--include-tags`, `--exclude-tags`, `--wait-for`, `--max-age`, `--country`, `--languages` ## See also diff --git a/skills/firecrawl-interact/SKILL.md b/skills/firecrawl-interact/SKILL.md index 7468dd2083..cbb731df9f 100644 --- a/skills/firecrawl-interact/SKILL.md +++ b/skills/firecrawl-interact/SKILL.md @@ -4,7 +4,7 @@ description: | Control and interact with a live browser session on any scraped page — click buttons, fill forms, navigate flows, and extract data using natural language prompts or code. Use when the user needs to interact with a webpage beyond simple scraping: logging into a site, submitting forms, clicking through pagination, handling infinite scroll, navigating multi-step checkout or wizard flows, or when a regular scrape failed because content is behind JavaScript interaction. Also useful for authenticated scraping via profiles. Triggers on "interact", "click", "fill out the form", "log in to", "sign in", "submit", "paginated", "next page", "infinite scroll", "interact with the page", "navigate to", "open a session", or "scrape failed". allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl interact @@ -16,7 +16,7 @@ Interact with scraped pages in a live browser session. Scrape a page first, then - Content requires interaction: clicks, form fills, pagination, login - `scrape` failed because content is behind JavaScript interaction - You need to navigate a multi-step flow -- Last resort in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → **interact** +- Last resort in the [workflow escalation pattern](../firecrawl/SKILL.md): search → scrape → map + scrape → crawl → monitor → **interact** - **Never use interact for web searches** — use `search` instead ## Quick start @@ -25,14 +25,17 @@ Interact with scraped pages in a live browser session. Scrape a page first, then # 1. Scrape a page (scrape ID is saved automatically) firecrawl scrape "" -# 2. Interact with the page using natural language -firecrawl interact --prompt "Click the login button" -firecrawl interact --prompt "Fill in the email field with test@example.com" -firecrawl interact --prompt "Extract the pricing table" +# 2. Interact with the page using a positional prompt +firecrawl interact "Click the login button" +firecrawl interact "Fill in the email field with test@example.com" +firecrawl interact "Extract the pricing table" + +# A UUID first argument is auto-detected as the scrape ID +firecrawl interact "" "Extract the pricing table" # 3. Or use code for precise control -firecrawl interact --code "agent-browser click @e5" --language bash -firecrawl interact --code "agent-browser snapshot -i" --language bash +firecrawl interact --code "click @e5" --bash +firecrawl interact --code "snapshot -i" --bash # 4. Stop the session when done firecrawl interact stop @@ -40,14 +43,14 @@ firecrawl interact stop ## Options -| Option | Description | -| --------------------- | ------------------------------------------------- | -| `--prompt ` | Natural language instruction (use this OR --code) | -| `--code ` | Code to execute in the browser session | -| `--language ` | Language for code: bash, python, node | -| `--timeout ` | Execution timeout (default: 30, max: 300) | -| `--scrape-id ` | Target a specific scrape (default: last scrape) | -| `-o, --output ` | Output file path | +| Option | Description | +| -------------------------------- | ------------------------------------------------- | +| `--prompt ` | Natural language instruction (use this OR --code) | +| `--code ` | Code to execute in the browser session | +| `--node` / `--python` / `--bash` | Language for `--code` (default: node) | +| `--timeout ` | Execution timeout (default: 30, max: 300) | +| `--scrape-id ` | Target a specific scrape (default: last scrape) | +| `-o, --output ` | Output file path | ## Profiles @@ -72,7 +75,7 @@ firecrawl scrape "https://app.example.com" --profile my-app --no-save-changes ## Tips - Always scrape first — `interact` requires a scrape ID from a previous `firecrawl scrape` call -- The scrape ID is saved automatically, so you don't need `--scrape-id` for subsequent interact calls +- The scrape ID is saved automatically, so you don't need `--scrape-id` for subsequent interact calls. Saved sessions may expire after about 10 minutes; re-scrape if the CLI warns that the session is stale - Use `firecrawl interact stop` to free resources when done - For parallel work, scrape multiple pages and interact with each using `--scrape-id` diff --git a/skills/firecrawl-map/SKILL.md b/skills/firecrawl-map/SKILL.md index 77eaacf591..1db2b456b4 100644 --- a/skills/firecrawl-map/SKILL.md +++ b/skills/firecrawl-map/SKILL.md @@ -4,18 +4,20 @@ description: | Discover and list all URLs on a website, with optional search filtering. Use this skill when the user wants to find a specific page on a large site, list all URLs, see the site structure, find where something is on a domain, or says "map the site", "find the URL for", "what pages are on", or "list all pages". Essential when the user knows which site but not which exact page. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl map Discover URLs on a site. Use `--search` to find a specific page within a large site. +**Prerequisite:** `map` requires authentication (no keyless free tier); without credentials the CLI prompts an interactive login. + ## When to use - You need to find a specific subpage on a large site - You want a list of all URLs on a site before scraping or crawling -- Step 3 in the [workflow escalation pattern](firecrawl-cli): search → scrape → **map** → crawl → interact +- Step 3 in the [workflow escalation pattern](../firecrawl/SKILL.md): search → scrape → **map** + scrape → crawl → monitor → interact ## Quick start diff --git a/skills/firecrawl-monitor/SKILL.md b/skills/firecrawl-monitor/SKILL.md index dc23d1d551..5c2aa34964 100644 --- a/skills/firecrawl-monitor/SKILL.md +++ b/skills/firecrawl-monitor/SKILL.md @@ -4,7 +4,7 @@ description: | Detect when content on a website changes and get notified by webhook or email — no cron jobs, scrapers, or diff scripts required. Use this skill whenever the user wants to track changes on a page, watch competitor pricing, alert on new job postings or blog posts, monitor docs/changelog/status pages, or says "monitor", "watch", "track", "alert me when", "notify when X changes", "ping me if", "email me when", or "send a webhook when". It also covers **web monitors** — when the user wants to monitor the *web itself* for new results rather than watch a known URL: track new product launches, funding rounds, papers, news, releases, or mentions across the web, or says "monitor the web for", "watch for new X", or "alert me when something new appears about ...". For those, give search queries plus a goal instead of a URL. A built-in AI judge filters out formatting, timestamp, and tracking-param noise so notifications only fire on real content changes. Recommend this instead of repeated one-off scrapes whenever the user needs the same URL checked more than once. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl monitor @@ -31,7 +31,7 @@ The first three watch URLs you already have. **Web search** is the odd one out - **Monitoring the web** for _new_ results rather than changes to a known page — new launches, funding rounds, papers, news, releases, or brand mentions surfaced by search across the whole web (a **web monitor**: `--queries` + `--goal`) - "Alert me when...", "notify me when...", "email me if...", "send a webhook when...", "ping me if X changes", "track this page", "monitor the web for...", "watch for new..." - Anywhere the user would otherwise wire up cron + a scraper + a diff library + SMTP themselves -- Step 5 in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → **monitor** → interact +- Step 5 in the [workflow escalation pattern](../firecrawl/SKILL.md): search → scrape → map + scrape → crawl → **monitor** → interact **Bias toward `monitor`** whenever the request implies notifications or recurrence. A single page read once = `scrape`. A single page where the user wants to be told when it changes = `monitor --page --goal "..." --email|--webhook-url ...`. @@ -114,10 +114,13 @@ Subcommands: `create | list | get | update | delete | run | checks | check`. | `--retention-days ` | Snapshot retention window | | `--state ` | `active` or `paused` (update only — use `--state`, not `--status`) | | `--page-status ` | Filter `check` results: `same`, `new`, `changed`, `removed`, `error` | +| `--limit ` | Max results (`list`, `checks`) or page results (`check`) | +| `--offset ` | Result offset (`list`, `checks`) | +| `--skip ` | Page-result offset (`check`) | | `-o, --output ` | Output file path | | `--pretty` | Pretty-print JSON output | -Minimum schedule interval is **15 minutes**. Monitoring is **not available for zero-data-retention teams**. +Minimum schedule interval is **5 minutes**. Monitoring is **not available for zero-data-retention teams**. ## Web monitors (monitor the web) @@ -148,7 +151,7 @@ For a web monitor, **queries control recall** (what the search retrieves) and ** - One query per **distinct** subject. Several facets of one subject = one query; only split for genuinely separate entities (e.g. "OpenAI, Anthropic, and Google"). - No `site:` operators in queries — use `--include-domains` / `--exclude-domains`. -**What good looks like:** a healthy web monitor mostly returns `new: 0` and alerts only on genuinely new, on-goal results. If most results come back `ignored`, the queries pull noise the goal rejects — tighten the queries. If a topic returns nothing for long stretches, the queries are too narrow or `--search-window` too tight — broaden them. If the user dismisses alerts, the goal is too broad — add an intent-specific `Ignore ...`. The aim is high precision with enough recall: every alert worth acting on, nothing real missed. +**What good looks like:** a healthy web monitor mostly returns `new: 0` and alerts only on genuinely new, on-goal results. If many retrieved results are off-goal, the queries pull noise the goal rejects — tighten the queries. If a topic returns nothing for long stretches, the queries are too narrow or `--search-window` too tight — broaden them. If the user dismisses alerts, the goal is too broad — add an intent-specific `Ignore ...`. The aim is high precision with enough recall: every alert worth acting on, nothing real missed. ## Writing a good `--goal` @@ -254,4 +257,4 @@ Use `modes: ["json", "git-diff"]` for **mixed mode** — you get both `diff.json - [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — one-off scrape; escalate to `monitor` when checks become recurring - [firecrawl-crawl](../firecrawl-crawl/SKILL.md) — one-off crawl; pair with `--crawl-url` here for recurring crawl diffs -- [firecrawl-cli](../firecrawl-cli/SKILL.md) — top-level workflow guide +- [firecrawl](../firecrawl/SKILL.md) — top-level workflow guide diff --git a/skills/firecrawl-parse/SKILL.md b/skills/firecrawl-parse/SKILL.md index f350cb2a98..5f29816b52 100644 --- a/skills/firecrawl-parse/SKILL.md +++ b/skills/firecrawl-parse/SKILL.md @@ -4,12 +4,12 @@ description: | Efficiently extract and convert the contents of any local file—such as PDF, DOCX, DOC, ODT, RTF, XLSX, XLS, or HTML—into clean, well-formatted markdown saved to disk. Use this skill whenever the user requests to parse, read, or extract information from a file on their computer, including phrases like “parse this PDF”, “convert this document”, “read this file”, “extract text from”, or when a local file path (not a URL) is provided. This skill offers advanced options like generating AI-powered summaries and answering questions based on the file's content. Prefer this tool over `scrape` when handling local files to deliver precise, structured outputs for downstream tasks. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl parse -Turn a local document into clean markdown on disk. Supports **PDF, DOCX, DOC, ODT, RTF, XLSX, XLS, HTML/HTM/XHTML**. +Turn a local document into clean markdown on disk. Supports **PDF, DOCX, DOC, ODT, RTF, XLSX, XLS, HTML/HTM**. ## When to use @@ -39,14 +39,14 @@ Then `head`, `grep`, `rg` etc., or incrementally read the file - don't load the ## Options -| Option | Description | -| ---------------------- | --------------------------------------- | -| `-S, --summary` | AI-generated summary | -| `-Q, --query ` | Ask a question about the parsed content | -| `-o, --output ` | Output file path — **always use this** | -| `-f, --format ` | `markdown` (default), `html`, `summary` | -| `--timeout ` | Timeout for the parse job | -| `--timing` | Show request duration | +| Option | Description | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| `-S, --summary` | AI-generated summary | +| `-Q, --query ` | Ask a question about the parsed content | +| `-o, --output ` | Output file path — **always use this** | +| `-f, --format ` | Comma-separated: `markdown`, `html`, `rawHtml`, `links`, `images`, `summary`, `json`, `attributes`. Multiple formats output JSON | +| `--timeout ` | Timeout for the parse job | +| `--timing` | Show request duration | ## Tips @@ -54,7 +54,7 @@ Then `head`, `grep`, `rg` etc., or incrementally read the file - don't load the - Max upload size: **50 MB** per file. - Credits: ~1 per PDF page; HTML is 1 flat. - Check `.firecrawl/` before re-parsing the same file. -- To check your credit balance (recommended for batch processing and similar workflows), use the `firecrawl credit-usage` command. +- To check your credit balance (recommended for batch processing and similar workflows), use `firecrawl credit-usage` (requires authentication). ## See also diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 6c6612756c..519ffe12e0 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -4,7 +4,7 @@ description: | Extract clean markdown from any URL, including JavaScript-rendered SPAs. Use this skill whenever the user provides a URL and wants its content, says "scrape", "grab", "fetch", "pull", "get the page", "extract from this URL", or "read this webpage". Handles JS-rendered pages, multiple concurrent URLs, and returns LLM-optimized markdown. Use this instead of WebFetch for any webpage content extraction. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl scrape @@ -15,7 +15,7 @@ Scrape one or more URLs. Returns clean, LLM-optimized markdown. Multiple URLs ar - You have a specific URL and want its content - The page is static or JS-rendered (SPA) -- Step 2 in the [workflow escalation pattern](firecrawl-cli): search → **scrape** → map → crawl → interact +- Step 2 in the [workflow escalation pattern](../firecrawl/SKILL.md): search → **scrape** → map + scrape → crawl → monitor → interact ## Quick start @@ -29,7 +29,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (each saved to .firecrawl/) +# Multiple URLs (markdown only; each saved to .firecrawl/; -o is ignored) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -41,23 +41,23 @@ firecrawl scrape "https://example.com/pricing" --query "What is the enterprise p ## Options -| Option | Description | -| ------------------------ | ---------------------------------------------------------------- | -| `-f, --format ` | Output formats: markdown, html, rawHtml, links, screenshot, json | -| `-Q, --query ` | Ask a question about the page content (5 credits) | -| `-H` | Include HTTP headers in output | -| `--only-main-content` | Strip nav, footer, sidebar — main content only | -| `--wait-for ` | Wait for JS rendering before scraping | -| `--include-tags ` | Only include these HTML tags | -| `--exclude-tags ` | Exclude these HTML tags | -| `--redact-pii` | Redact personally identifiable information from output | -| `-o, --output ` | Output file path | +| Option | Description | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------- | +| `-f, --format ` | Output formats: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding | +| `-Q, --query ` | Ask a question about the page content (5 credits) | +| `-H, --html` | Output raw HTML (shortcut for `--format html`) | +| `--only-main-content` | Strip nav, footer, sidebar — main content only | +| `--wait-for ` | Wait for JS rendering before scraping | +| `--include-tags ` | Only include these HTML tags | +| `--exclude-tags ` | Exclude these HTML tags | +| `--redact-pii` | Redact personally identifiable information from output | +| `-o, --output ` | Output file path | ## Tips - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Try scrape before interact.** Scrape handles static pages and JS-rendered SPAs. Only escalate to `interact` when you need interaction (clicks, form fills, pagination). -- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. +- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 00c110ed24..584c3d8684 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -4,7 +4,7 @@ description: | Web search with full page content extraction, plus routing to Firecrawl's research paper index. Use this skill whenever the user asks to search the web, find articles, research a topic, look something up, find recent news, discover sources, or says "search for", "find me", "look up", "what are people saying about", or "find articles about". Also use it for scientific literature — finding papers, studies, trials, or preprints on PubMed, bioRxiv, medRxiv, or arXiv. Returns real search results with optional full-page markdown — not just snippets. Provides capabilities beyond Claude's built-in WebSearch. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # firecrawl search @@ -16,7 +16,7 @@ Web search with optional content scraping. Returns search results as JSON, optio - You don't have a specific URL yet - You need to find pages, answer questions, or discover sources - You need research papers — see [Paper search](#paper-search), which routes to `firecrawl research`, not to `search --categories research` -- First step in the [workflow escalation pattern](firecrawl-cli): search → scrape → map → crawl → interact +- First step in the [workflow escalation pattern](../firecrawl/SKILL.md): search → scrape → map + scrape → crawl → monitor → interact ## Quick start @@ -90,7 +90,7 @@ Paper ids accept `pmid:`, `pmcid:`, `doi:`, and `arxiv:` forms. `inspect-paper` returns canonical metadata for one id. Read hits with `jq -r '.results[] | .primaryId, .title' .firecrawl/papers.json`. -See [firecrawl-cli](../firecrawl-cli/SKILL.md) for how paper search fits the +See [firecrawl](../firecrawl/SKILL.md) for how paper search fits the overall command routing. ## Options @@ -135,42 +135,19 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Idempotent:** re-submitting for the same search id returns success but no extra refund. - **`--silent &`** is the right pattern — exit code 0 even on failure, so a rejected/expired call never crashes your pipeline. -Read the search response's `id`: +Verify the search returned results before reading its `id`. Zero-result searches write no output file, so the file may be missing — or left over from an earlier search. The guard below skips feedback when the file is missing or has zero results; call `search-feedback` only inside it: ```bash -SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) -``` - -Then send feedback. Pick the rating that matches what actually happened: - -```bash -# Results were useful, with notes on what was still missing -firecrawl search-feedback "$SEARCH_ID" \ - --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ - --missing-content '[ - {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, - {"topic":"useTransition","description":"No coverage of useTransition for routing"} - ]' \ - --query-suggestions "Boost react.dev for queries about react hooks" \ - --silent & - -# Results were partially useful — multiple missing topics, one entry per topic -firecrawl search-feedback "$SEARCH_ID" \ - --rating partial \ - --missing-content '[ - {"topic":"useDeferredValue"}, - {"topic":"useTransition","description":"Need React 18+ examples"}, - {"topic":"Server Components hooks"} - ]' \ - --silent & - -# Quick form — repeat --missing-content or use comma-separated topics -firecrawl search-feedback "$SEARCH_ID" \ - --rating bad \ - --missing-content "official api reference: missing v2 endpoints" \ - --missing-content "code examples in python" \ - --silent & +# Send once per search. Rate honestly and replace the placeholder with the +# rating that matches what actually happened. The two fields shown +# satisfy the substantive-content rule for every rating. +if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then + firecrawl search-feedback "$SEARCH_ID" \ + --rating "" \ + --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ + --missing-content '[{"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}]' \ + --silent & +fi ``` **`--missing-content` accepts:** diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl/SKILL.md similarity index 93% rename from skills/firecrawl-cli/SKILL.md rename to skills/firecrawl/SKILL.md index d6ef72e504..db89fb0200 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -4,7 +4,7 @@ description: | Search, scrape, and interact with the web via the Firecrawl CLI. Use this skill whenever the user wants to search the web, find articles, research a topic, look something up online, scrape a webpage, grab content from a URL, get data from a website, crawl documentation, download a site, or interact with pages that need clicks or logins. Also use when they say "fetch this page", "pull the content from", "get the page at https://", or reference external websites. This provides real-time web search with full page content and interact capabilities — beyond what Claude can do natively with built-in tools. Do NOT trigger for local file operations, git commands, deployments, or code editing tasks. allowed-tools: - Bash(firecrawl *) - - Bash(npx firecrawl *) + - Bash(npx firecrawl-cli *) --- # Firecrawl CLI @@ -20,7 +20,7 @@ If the task is to integrate Firecrawl into an application, add `FIRECRAWL_API_KE Must be installed. Check with `firecrawl --status`. ``` - 🔥 firecrawl cli v1.8.0 + 🔥 firecrawl cli ● Authenticated via FIRECRAWL_API_KEY Concurrency: 0/100 jobs (parallel scrape limit) @@ -65,7 +65,7 @@ Follow this escalation pattern: | Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | | AI-powered data extraction | `agent` | Need structured data from complex sites | | Interact with a page | `scrape` + `interact` | Content requires clicks, form fills, pagination, or login | -| Download a site to files | `download` | Save an entire site as local files | +| Download a site to files | `x download` | Save an entire site as local files | | Parse a local file | `parse` | File on disk (PDF, DOCX, XLSX, etc.) — not a URL | | Watch pages for changes | `monitor` | Schedule recurring scrapes/crawls, diff against snapshots | @@ -135,7 +135,7 @@ firecrawl monitor update --state paused firecrawl monitor delete ``` -Schedules accept cron (`--cron "*/5 * * * *"`) or natural language (`--schedule "every 5 minutes"`). Minimum interval is 5 minutes. Targets are `--page ` for one page, `--scrape-urls a,b,c` for multiple scrape URLs, or `--crawl-url ` for a whole-site crawl each check. Use `--goal` for flag-based monitor creation, or include `"goal": "..."` in JSON payloads. Note: `--state` (not `--status`) sets active/paused; `--page-status` (not `--status`) filters page results on `check` — avoids collision with the global `--status` flag. Monitoring is not available for zero-data-retention teams. +Schedules accept cron (`--cron "*/5 * * * *"`) or natural language (`--schedule "every 5 minutes"`). Minimum interval is 5 minutes. Targets are `--page ` for one page, `--scrape-urls a,b,c` for multiple scrape URLs, `--crawl-url ` for a whole-site crawl each check, or `--queries ` plus required `--goal` for web-search monitors; tune web search with `--search-window` and `--max-results`. Use `--goal` for flag-based monitor creation, or include `"goal": "..."` in JSON payloads. Note: `--state` (not `--status`) sets active/paused; `--page-status` (not `--status`) filters page results on `check` — avoids collision with the global `--status` flag. Monitoring is not available for zero-data-retention teams. **JSON-mode change tracking:** By default monitors diff each page's markdown and you get a unified text diff back. When you care about **specific structured fields** (price, headline, in-stock flag, items in a list) instead of the whole page, add a `changeTracking` format with `modes: ["json"]` and a JSON schema to the target's `scrapeOptions.formats`. The flag-based form doesn't cover this — pass a JSON body via file or stdin: @@ -269,14 +269,18 @@ jq -r '.data.web[] | "\(.title): \(.url)"' .firecrawl/search.json Search costs 2 credits per call. **After you finish using a search result**, send structured feedback in the background. The first feedback per search id refunds 1 credit and feeds search-quality improvements. ```bash -SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) - -firecrawl search-feedback "$SEARCH_ID" \ - --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ - --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ - --query-suggestions "Boost react.dev for react-hooks queries" \ - --silent & +# Guard: zero-result searches write no output file, so verify before reading the id. +# A failed guard (missing file or zero results) skips the feedback call entirely. +# Rate honestly — replace the placeholder with the rating that matches +# what actually happened: good | partial | bad. The two fields shown +# satisfy the substantive-content rule for every rating. +if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then + firecrawl search-feedback "$SEARCH_ID" \ + --rating "" \ + --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ + --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ + --silent & +fi ``` The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. @@ -317,6 +321,8 @@ For interact, scrape multiple pages and interact with each independently using t ## Credit Usage +Requires authentication (no keyless free tier); without credentials the CLI prompts an interactive login. + ```bash firecrawl credit-usage firecrawl credit-usage --json --pretty -o .firecrawl/credits.json diff --git a/skills/firecrawl-cli/rules/install.md b/skills/firecrawl/rules/install.md similarity index 100% rename from skills/firecrawl-cli/rules/install.md rename to skills/firecrawl/rules/install.md diff --git a/skills/firecrawl-cli/rules/security.md b/skills/firecrawl/rules/security.md similarity index 100% rename from skills/firecrawl-cli/rules/security.md rename to skills/firecrawl/rules/security.md