diff --git a/README.md b/README.md index 4c2c637570..ea64d22dae 100644 --- a/README.md +++ b/README.md @@ -370,20 +370,21 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| -------------------------------------- | -------------------------------------------------------------------------- | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--valuable-results ` | Search only: every useful result as `source:position`, e.g. `web:1,news:2` | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index 5494c1e3cb..a51b2a7eac 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -193,9 +193,7 @@ The `check` response then carries a per-field diff (paths like `plans[0].price`) }, "snapshot": { "json": { - "plans": [ - /* current full extraction */ - ] + "plans": [/* current full extraction */] } } } @@ -255,11 +253,12 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li These patterns are useful when working with file-based output (`-o` flag) for complex tasks: ```bash -# Extract URLs from search -jq -r '.data.web[].url' .firecrawl/search.json +# Extract URLs with their source and 1-indexed position (needed for feedback). +# Each group is numbered from 1 independently, so keep the source with it. +jq -r '.data | to_entries[] | .key as $s | .value | to_entries[] | "\($s):\(.key + 1)\t\(.value.url)"' .firecrawl/search.json -# Get titles and URLs -jq -r '.data.web[] | "\(.title): \(.url)"' .firecrawl/search.json +# Web results only, with positions, titles, and URLs +jq -r '.data.web | to_entries[] | "web:\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json ``` ## After search: send feedback (refunds 1 credit) @@ -271,13 +270,13 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ + --valuable-results "web:1,web:3" \ --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ --query-suggestions "Boost react.dev for react-hooks queries" \ --silent & ``` -The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. +The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-results` as `source:position` (e.g. `web:1,news:2`) — results come back grouped and each group is numbered from 1, so the source is required. List **every** useful one; unlisted results count as not useful. Reserve `--valuable-sources` for useful URLs that were not among the returned results. **Opt out:** `export FIRECRAWL_NO_SEARCH_FEEDBACK=1` makes the CLI skip every feedback call silently. Respect that flag — do not try to work around it. See [firecrawl-search](../firecrawl-search/SKILL.md) for the full pattern. diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 87b426cf92..907706e55b 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -49,7 +49,7 @@ firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json - **`--scrape` fetches full content** — don't re-scrape URLs from search results. This saves credits and avoids redundant fetches. - Always write results to `.firecrawl/` with `-o` to avoid context window bloat. -- Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` +- Use `jq` to extract URLs **with their source and 1-indexed position** (you'll need both for feedback): `jq -r '.data | to_entries[] | .key as $s | .value | to_entries[] | "\($s):\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` ## Send feedback after every search (refunds 1 credit) @@ -62,9 +62,10 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Time window:** must be sent within ~2 minutes of the search. Late feedback is rejected. - **`--missing-content` is the most important field.** It's a list of _specific pieces_ of content you expected but did not find. One topic per entry — do not pack multiple topics into one string. These aggregate across teams and tell us what to index next. +- **`--valuable-results` marks which results were useful.** Results come back grouped (`data.web`, `data.images`, `data.news`) and **each group is numbered from 1 independently**, so every entry is `source:position` — `web:1` and `news:1` are two different results. **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT among the returned results (e.g. a page you found by following a result's link) — never report the same result in both. - **Substantive content required** (zero-effort feedback is rejected with HTTP 400): - - `good` → must include at least one `--valuable-sources` entry. - - `partial` → must include `--valuable-sources` or `--missing-content`. + - `good` → must include `--valuable-results` or at least one `--valuable-sources` entry. + - `partial` → must include `--valuable-results`, `--valuable-sources`, or `--missing-content`. - `bad` → must include `--missing-content` or `--query-suggestions`. - **Daily refund cap (per team, per UTC day, default 100 credits).** Once your team has been refunded 100 credits today, further submissions still record feedback but no longer refund credits. The response includes `creditsRefundedToday` / `dailyRefundCap` / `dailyCapReached`. **When `dailyCapReached: true`, stop calling `search-feedback` for the rest of the UTC day** — it won't refund anything and you're wasting bandwidth. - **Idempotent:** re-submitting for the same search id returns success but no extra refund. @@ -79,10 +80,10 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) Then send feedback. Pick the rating that matches what actually happened: ```bash -# Results were useful, with notes on what was still missing +# Results were useful — web positions 1 and 3 answered the question firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ + --valuable-results "web:1,web:3" \ --missing-content '[ {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, {"topic":"useTransition","description":"No coverage of useTransition for routing"} diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index cbb906ace7..e152a284c6 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -6,6 +6,7 @@ import { parseFeedbackListArg, parsePageNumbersArg, } from '../../commands/feedback'; +import { parseValuableResultsArg } from '../../commands/search-feedback'; import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; @@ -207,4 +208,53 @@ describe('feedback parsing', () => { expect(parsePageNumbersArg('1, 2, bad, -1, 3')).toEqual([1, 2, 3]); expect(parsePageNumbersArg('[4,5]')).toEqual([4, 5]); }); + + it('parses valuable results as source:position pairs', () => { + expect(parseValuableResultsArg('web:1, news:2')).toEqual([ + { source: 'web', position: 1 }, + { source: 'news', position: 2 }, + ]); + expect(parseValuableResultsArg('images:3')).toEqual([ + { source: 'images', position: 3 }, + ]); + }); + + it('parses valuable results from JSON, keeping reasons', () => { + expect( + parseValuableResultsArg( + '[{"source":"web","position":1,"reason":"Answered it"},{"source":"news","position":2}]' + ) + ).toEqual([ + { source: 'web', position: 1, reason: 'Answered it' }, + { source: 'news', position: 2 }, + ]); + }); + + // Each group is numbered from 1 independently, so a bare position does not + // identify a result. + it('rejects valuable results without a source', () => { + expect(() => parseValuableResultsArg('1,3')).toThrow( + 'must be "source:position"' + ); + expect(() => parseValuableResultsArg('[{"position":1}]')).toThrow( + 'source must be one of' + ); + }); + + it('rejects unknown sources and non-positive positions', () => { + expect(() => parseValuableResultsArg('video:1')).toThrow( + 'source must be one of' + ); + expect(() => parseValuableResultsArg('web:0')).toThrow( + 'positions must be integers of 1 or greater' + ); + expect(() => parseValuableResultsArg('web:abc')).toThrow( + 'positions must be integers of 1 or greater' + ); + }); + + it('returns undefined for empty input', () => { + expect(parseValuableResultsArg(undefined)).toBeUndefined(); + expect(parseValuableResultsArg(' ')).toBeUndefined(); + }); }); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 14318a8c9e..96b48f01f9 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -4,9 +4,11 @@ import { getConfig, isCustomApiUrl, validateConfig } from '../utils/config'; import { getClient } from '../utils/client'; import { parseMissingContentArg, + parseValuableResultsArg, parseValuableSourcesArg, type MissingContentInput, type SearchFeedbackRating, + type ValuableResultInput, type ValuableSourceInput, } from './search-feedback'; @@ -20,6 +22,7 @@ export interface EndpointFeedbackOptions { tags?: string[]; note?: string; valuableSources?: ValuableSourceInput[]; + valuableResults?: ValuableResultInput[]; missingContent?: MissingContentInput[]; querySuggestions?: string; url?: string; @@ -207,6 +210,7 @@ export function parseEndpointFeedbackCliOptions(options: { metadata?: string; metadataFile?: string; valuableSources?: string; + valuableResults?: string; missingContent?: string | string[]; rating?: string; }) { @@ -217,6 +221,7 @@ export function parseEndpointFeedbackCliOptions(options: { pageNumbers: parsePageNumbersArg(options.pageNumbers), metadata: parseMetadataArg(options.metadata, options.metadataFile), valuableSources: parseValuableSourcesArg(options.valuableSources), + valuableResults: parseValuableResultsArg(options.valuableResults), missingContent: parseMissingContentArg(options.missingContent), }; } @@ -261,6 +266,7 @@ export async function executeEndpointFeedback( ['tags', normalizeList(options.tags)], ['note', options.note], ['valuableSources', options.valuableSources], + ['valuableResults', options.valuableResults], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], ['url', options.url], diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index 35898e78b4..f99994766a 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -13,10 +13,28 @@ export interface MissingContentInput { description?: string; } +export type SearchResultSource = 'web' | 'images' | 'news'; + +export const SEARCH_RESULT_SOURCES: readonly SearchResultSource[] = [ + 'web', + 'images', + 'news', +]; + +// Search results come back grouped — data.web, data.images, data.news — and +// each group is numbered from 1 independently, so a position is only +// meaningful alongside the group it indexes into. +export interface ValuableResultInput { + source: SearchResultSource; + position: number; + reason?: string; +} + export interface SearchFeedbackOptions { searchId: string; rating: SearchFeedbackRating; valuableSources?: ValuableSourceInput[]; + valuableResults?: ValuableResultInput[]; missingContent?: MissingContentInput[]; querySuggestions?: string; apiKey?: string; @@ -118,6 +136,9 @@ export async function executeSearchFeedback( ...(s.reason ? { reason: s.reason } : {}), })); } + if (options.valuableResults && options.valuableResults.length > 0) { + body.valuableResults = options.valuableResults; + } if (options.missingContent && options.missingContent.length > 0) { body.missingContent = options.missingContent .filter((m) => !!m.topic) @@ -350,6 +371,95 @@ export function parseValuableSourcesArg( .map((url) => ({ url })); } +function isSearchResultSource(value: unknown): value is SearchResultSource { + return ( + typeof value === 'string' && + (SEARCH_RESULT_SOURCES as readonly string[]).includes(value) + ); +} + +function parsePositionValue(raw: unknown, flag: string): number { + const position = typeof raw === 'string' ? Number(raw.trim()) : raw; + if ( + typeof position !== 'number' || + !Number.isInteger(position) || + position < 1 + ) { + throw new Error(`${flag} positions must be integers of 1 or greater.`); + } + return position; +} + +// Accepts a compact "source:position" list (e.g. "web:1,news:2") or a JSON +// array of {source, position, reason} entries. The source is always required: +// results are grouped and each group is numbered from 1, so a bare position +// does not identify a result. +export function parseValuableResultsArg( + raw: string | undefined, + flag = '--valuable-results' +): ValuableResultInput[] | undefined { + if (!raw) return undefined; + const trimmed = raw.trim(); + if (!trimmed) return undefined; + + const sourceList = SEARCH_RESULT_SOURCES.join(' | '); + + if (trimmed.startsWith('[') || trimmed.startsWith('{')) { + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new Error( + `${flag} must be valid JSON or a comma-separated "source:position" list.` + ); + } + + const entries = Array.isArray(parsed) ? parsed : [parsed]; + const cleaned = entries.map((entry: any) => { + if (!entry || typeof entry !== 'object') { + throw new Error( + `${flag} JSON entries must be objects with a source and a position.` + ); + } + if (!isSearchResultSource(entry.source)) { + throw new Error(`${flag} source must be one of: ${sourceList}.`); + } + return { + source: entry.source, + position: parsePositionValue(entry.position, flag), + ...(typeof entry.reason === 'string' && entry.reason.trim() + ? { reason: entry.reason } + : {}), + }; + }); + return cleaned.length > 0 ? cleaned : undefined; + } + + const cleaned = trimmed + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => { + const separator = entry.lastIndexOf(':'); + if (separator === -1) { + throw new Error( + `${flag} entries must be "source:position" (e.g. web:1) — ` + + `results are grouped, so a bare position is ambiguous.` + ); + } + const source = entry.slice(0, separator).trim(); + if (!isSearchResultSource(source)) { + throw new Error(`${flag} source must be one of: ${sourceList}.`); + } + return { + source, + position: parsePositionValue(entry.slice(separator + 1), flag), + }; + }); + + return cleaned.length > 0 ? cleaned : undefined; +} + // Accepts JSON arrays/objects, "topic: description" strings, comma- // separated topic lists, or repeated values. Caps at 20 entries. export function parseMissingContentArg( diff --git a/src/index.ts b/src/index.ts index 5c1ae11f3f..2983dc76d9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,7 @@ import { import { handleSearchFeedbackCommand, parseValuableSourcesArg, + parseValuableResultsArg, parseMissingContentArg, type SearchFeedbackRating, } from './commands/search-feedback'; @@ -1294,9 +1295,18 @@ function createSearchFeedbackCommand(): Command { ) .argument('', 'The id returned by `firecrawl search ... --json`') .requiredOption('--rating ', 'Overall rating: good | bad | partial') + .option( + '--valuable-results ', + 'Every result that was useful, as "source:position" (e.g. ' + + '"web:1,news:2") OR a JSON array of {source, position, reason}. ' + + 'Results are grouped and each group is numbered from 1, so the ' + + 'source is required. Unlisted results are treated as not useful.' + ) .option( '--valuable-sources ', - 'Comma-separated URLs OR JSON array of {url, reason} entries' + 'Comma-separated URLs OR JSON array of {url, reason} entries. ' + + 'For useful URLs NOT among the returned results; use ' + + '--valuable-results for results the search returned.' ) .option( '--missing-content ', @@ -1337,6 +1347,14 @@ function createSearchFeedbackCommand(): Command { process.exit(1); } + let valuableResults; + try { + valuableResults = parseValuableResultsArg(options.valuableResults); + } catch (error: any) { + console.error('Error:', error?.message || 'Invalid --valuable-results'); + process.exit(1); + } + let missingContent; try { missingContent = parseMissingContentArg(options.missingContent); @@ -1349,6 +1367,7 @@ function createSearchFeedbackCommand(): Command { searchId, rating: rating as SearchFeedbackRating, valuableSources, + valuableResults, missingContent, querySuggestions: options.querySuggestions, apiKey: options.apiKey, @@ -1385,6 +1404,11 @@ function createFeedbackCommand(): Command { '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' ) + .option( + '--valuable-results ', + 'Search only: every useful result as "source:position" (e.g. ' + + '"web:1,news:2") OR a JSON array of {source, position, reason}' + ) .option( '--missing-content ', 'Specific pieces of content missing from results. ' + @@ -1440,6 +1464,7 @@ function createFeedbackCommand(): Command { tags: parsed.tags, note: options.note, valuableSources: parsed.valuableSources, + valuableResults: parsed.valuableResults, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, url: options.url,